wsl/hsakmt: initial commit
Signed-off-by: lyndonli <Lyndon.Li@amd.com> Signed-off-by: Horatio Zhang <Hongkun.Zhang@amd.com> Signed-off-by: Shi.Leslie <Yuliang.Shi@amd.com> Signed-off-by: LonglongYao <Longlong.Yao@amd.com> Signed-off-by: tiancyin <tianci.yin@amd.com> Signed-off-by: Frank Min <Frank.Min@amd.com> Signed-off-by: Aaron Liu <aaron.liu@amd.com> Signed-off-by: Shane Xiao <shane.xiao@amd.com> Signed-off-by: Lang Yu <lang.yu@amd.com> Signed-off-by: Feifei Xu <Feifei.Xu@amd.com> Signed-off-by: Ruili Ji <ruiliji2@amd.com> Signed-off-by: Qiang Yu <qiang.yu@amd.com> Signed-off-by: Flora Cui <flora.cui@amd.com>
Šī revīzija ir iekļauta:
Flora Cui
Frank Min
+118
@@ -0,0 +1,118 @@
|
||||
/*
|
||||
* Copyright © 2014 Advanced Micro Devices, Inc.
|
||||
*
|
||||
* Permission is hereby granted, free of charge, to any person
|
||||
* obtaining a copy of this software and associated documentation
|
||||
* files (the "Software"), to deal in the Software without
|
||||
* restriction, including without limitation the rights to use, copy,
|
||||
* modify, merge, publish, distribute, sublicense, and/or sell copies
|
||||
* of the Software, and to permit persons to whom the Software is
|
||||
* furnished to do so, subject to the following conditions:
|
||||
*
|
||||
* The above copyright notice and this permission notice (including
|
||||
* the next paragraph) shall be included in all copies or substantial
|
||||
* portions of the Software.
|
||||
*
|
||||
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
||||
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
||||
* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
||||
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
|
||||
* HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
|
||||
* WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
||||
* DEALINGS IN THE SOFTWARE.
|
||||
*/
|
||||
|
||||
#include <cassert>
|
||||
#include <cstring>
|
||||
|
||||
#include "libhsakmt.h"
|
||||
|
||||
static uint32_t runtime_capabilities_mask = 0;
|
||||
|
||||
HSAKMT_STATUS HSAKMTAPI hsaKmtDbgRegister(HSAuint32 NodeId) {
|
||||
CHECK_DXG_OPEN();
|
||||
return HSAKMT_STATUS_NOT_SUPPORTED;
|
||||
}
|
||||
|
||||
HSAKMT_STATUS HSAKMTAPI hsaKmtDbgUnregister(HSAuint32 NodeId) {
|
||||
CHECK_DXG_OPEN();
|
||||
return HSAKMT_STATUS_NOT_SUPPORTED;
|
||||
}
|
||||
|
||||
HSAKMT_STATUS HSAKMTAPI hsaKmtDbgWavefrontControl(
|
||||
HSAuint32 NodeId, HSA_DBG_WAVEOP Operand, HSA_DBG_WAVEMODE Mode,
|
||||
HSAuint32 TrapId, HsaDbgWaveMessage *DbgWaveMsgRing) {
|
||||
CHECK_DXG_OPEN();
|
||||
return HSAKMT_STATUS_NOT_SUPPORTED;
|
||||
}
|
||||
|
||||
HSAKMT_STATUS HSAKMTAPI hsaKmtDbgAddressWatch(
|
||||
HSAuint32 NodeId, HSAuint32 NumWatchPoints, HSA_DBG_WATCH_MODE WatchMode[],
|
||||
void *WatchAddress[], HSAuint64 WatchMask[], HsaEvent *WatchEvent[]) {
|
||||
CHECK_DXG_OPEN();
|
||||
return HSAKMT_STATUS_NOT_SUPPORTED;
|
||||
}
|
||||
|
||||
HSAKMT_STATUS HSAKMTAPI hsaKmtCheckRuntimeDebugSupport(void) {
|
||||
CHECK_DXG_OPEN();
|
||||
return HSAKMT_STATUS_NOT_SUPPORTED;
|
||||
}
|
||||
|
||||
HSAKMT_STATUS HSAKMTAPI hsaKmtRuntimeEnable(void *rDebug, bool setupTtmp) {
|
||||
HSAKMT_STATUS result = hsaKmtCheckRuntimeDebugSupport();
|
||||
|
||||
if (result)
|
||||
return result;
|
||||
|
||||
assert(false);
|
||||
return HSAKMT_STATUS_SUCCESS;
|
||||
}
|
||||
|
||||
HSAKMT_STATUS HSAKMTAPI hsaKmtRuntimeDisable(void) {
|
||||
HSAKMT_STATUS result = hsaKmtCheckRuntimeDebugSupport();
|
||||
|
||||
if (result)
|
||||
return result;
|
||||
|
||||
assert(false);
|
||||
return HSAKMT_STATUS_SUCCESS;
|
||||
}
|
||||
|
||||
HSAKMT_STATUS HSAKMTAPI hsaKmtGetRuntimeCapabilities(HSAuint32 *caps_mask) {
|
||||
CHECK_DXG_OPEN();
|
||||
*caps_mask = runtime_capabilities_mask;
|
||||
return HSAKMT_STATUS_SUCCESS;
|
||||
}
|
||||
|
||||
HSAKMT_STATUS HSAKMTAPI hsaKmtDbgEnable(void **runtime_info,
|
||||
HSAuint32 *data_size) {
|
||||
CHECK_DXG_OPEN();
|
||||
|
||||
return HSAKMT_STATUS_NOT_SUPPORTED;
|
||||
}
|
||||
HSAKMT_STATUS HSAKMTAPI hsaKmtDbgDisable(void) {
|
||||
CHECK_DXG_OPEN();
|
||||
return HSAKMT_STATUS_NOT_SUPPORTED;
|
||||
}
|
||||
|
||||
HSAKMT_STATUS HSAKMTAPI hsaKmtDbgGetDeviceData(void **data,
|
||||
HSAuint32 *n_entries,
|
||||
HSAuint32 *entry_size) {
|
||||
CHECK_DXG_OPEN();
|
||||
return HSAKMT_STATUS_NOT_SUPPORTED;
|
||||
}
|
||||
|
||||
HSAKMT_STATUS HSAKMTAPI hsaKmtDbgGetQueueData(void **data, HSAuint32 *n_entries,
|
||||
HSAuint32 *entry_size,
|
||||
bool suspend_queues) {
|
||||
CHECK_DXG_OPEN();
|
||||
return HSAKMT_STATUS_NOT_SUPPORTED;
|
||||
}
|
||||
|
||||
HSAKMT_STATUS HSAKMTAPI
|
||||
hsaKmtDebugTrapIoctl(struct kfd_ioctl_dbg_trap_args *args, HSA_QUEUEID *Queues,
|
||||
HSAuint64 *DebugReturn) {
|
||||
CHECK_DXG_OPEN();
|
||||
return HSAKMT_STATUS_NOT_SUPPORTED;
|
||||
}
|
||||
+129
@@ -0,0 +1,129 @@
|
||||
/*
|
||||
* Copyright © 2014 Advanced Micro Devices, Inc.
|
||||
*
|
||||
* Permission is hereby granted, free of charge, to any person
|
||||
* obtaining a copy of this software and associated documentation
|
||||
* files (the "Software"), to deal in the Software without
|
||||
* restriction, including without limitation the rights to use, copy,
|
||||
* modify, merge, publish, distribute, sublicense, and/or sell copies
|
||||
* of the Software, and to permit persons to whom the Software is
|
||||
* furnished to do so, subject to the following conditions:
|
||||
*
|
||||
* The above copyright notice and this permission notice (including
|
||||
* the next paragraph) shall be included in all copies or substantial
|
||||
* portions of the Software.
|
||||
*
|
||||
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
||||
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
||||
* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
||||
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
|
||||
* HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
|
||||
* WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
||||
* DEALINGS IN THE SOFTWARE.
|
||||
*/
|
||||
|
||||
#include <cstdio>
|
||||
#include <cassert>
|
||||
#include <thread>
|
||||
#include <chrono>
|
||||
#include "libhsakmt.h"
|
||||
|
||||
HSAKMT_STATUS HSAKMTAPI hsaKmtCreateEvent(HsaEventDescriptor *EventDesc,
|
||||
bool ManualReset, bool IsSignaled,
|
||||
HsaEvent **Event) {
|
||||
CHECK_DXG_OPEN();
|
||||
|
||||
assert(false);
|
||||
return HSAKMT_STATUS_SUCCESS;
|
||||
}
|
||||
|
||||
HSAKMT_STATUS HSAKMTAPI hsaKmtDestroyEvent(HsaEvent *Event) {
|
||||
CHECK_DXG_OPEN();
|
||||
|
||||
if (!Event)
|
||||
return HSAKMT_STATUS_INVALID_HANDLE;
|
||||
|
||||
assert(false);
|
||||
return HSAKMT_STATUS_SUCCESS;
|
||||
}
|
||||
|
||||
HSAKMT_STATUS HSAKMTAPI hsaKmtSetEvent(HsaEvent *Event) {
|
||||
CHECK_DXG_OPEN();
|
||||
|
||||
if (!Event)
|
||||
return HSAKMT_STATUS_INVALID_HANDLE;
|
||||
|
||||
assert(false);
|
||||
return HSAKMT_STATUS_SUCCESS;
|
||||
}
|
||||
|
||||
HSAKMT_STATUS HSAKMTAPI hsaKmtResetEvent(HsaEvent *Event) {
|
||||
CHECK_DXG_OPEN();
|
||||
|
||||
if (!Event)
|
||||
return HSAKMT_STATUS_INVALID_HANDLE;
|
||||
|
||||
assert(false);
|
||||
return HSAKMT_STATUS_SUCCESS;
|
||||
}
|
||||
|
||||
HSAKMT_STATUS HSAKMTAPI hsaKmtQueryEventState(HsaEvent *Event) {
|
||||
CHECK_DXG_OPEN();
|
||||
|
||||
if (!Event)
|
||||
return HSAKMT_STATUS_INVALID_HANDLE;
|
||||
|
||||
assert(false);
|
||||
return HSAKMT_STATUS_SUCCESS;
|
||||
}
|
||||
|
||||
HSAKMT_STATUS HSAKMTAPI hsaKmtWaitOnEvent(HsaEvent *Event,
|
||||
HSAuint32 Milliseconds) {
|
||||
return hsaKmtWaitOnEvent_Ext(Event, Milliseconds, NULL);
|
||||
}
|
||||
|
||||
HSAKMT_STATUS HSAKMTAPI hsaKmtWaitOnEvent_Ext(HsaEvent *Event,
|
||||
HSAuint32 Milliseconds,
|
||||
uint64_t *event_age) {
|
||||
if (!Event)
|
||||
return HSAKMT_STATUS_INVALID_HANDLE;
|
||||
|
||||
return hsaKmtWaitOnMultipleEvents_Ext(&Event, 1, true, Milliseconds,
|
||||
event_age);
|
||||
}
|
||||
|
||||
HSAKMT_STATUS HSAKMTAPI hsaKmtWaitOnMultipleEvents(HsaEvent *Events[],
|
||||
HSAuint32 NumEvents,
|
||||
bool WaitOnAll,
|
||||
HSAuint32 Milliseconds) {
|
||||
return hsaKmtWaitOnMultipleEvents_Ext(Events, NumEvents, WaitOnAll,
|
||||
Milliseconds, NULL);
|
||||
}
|
||||
|
||||
HSAKMT_STATUS HSAKMTAPI hsaKmtWaitOnMultipleEvents_Ext(HsaEvent *Events[],
|
||||
HSAuint32 NumEvents,
|
||||
bool WaitOnAll,
|
||||
HSAuint32 Milliseconds,
|
||||
uint64_t *event_age) {
|
||||
CHECK_DXG_OPEN();
|
||||
|
||||
if (!Events)
|
||||
return HSAKMT_STATUS_INVALID_HANDLE;
|
||||
|
||||
if (NumEvents == 1 && Events[0] == nullptr) {
|
||||
std::this_thread::sleep_for(std::chrono::microseconds(20));
|
||||
return HSAKMT_STATUS_SUCCESS;
|
||||
}
|
||||
|
||||
assert(false);
|
||||
return HSAKMT_STATUS_SUCCESS;
|
||||
}
|
||||
|
||||
HSAKMT_STATUS HSAKMTAPI hsaKmtOpenSMI(HSAuint32 NodeId, int *fd) {
|
||||
CHECK_DXG_OPEN();
|
||||
|
||||
pr_debug("[%s] node %d\n", __func__, NodeId);
|
||||
assert(false);
|
||||
return HSAKMT_STATUS_SUCCESS;
|
||||
}
|
||||
@@ -0,0 +1,47 @@
|
||||
/*
|
||||
* Copyright © 2014 Advanced Micro Devices, Inc.
|
||||
*
|
||||
* Permission is hereby granted, free of charge, to any person
|
||||
* obtaining a copy of this software and associated documentation
|
||||
* files (the "Software"), to deal in the Software without
|
||||
* restriction, including without limitation the rights to use, copy,
|
||||
* modify, merge, publish, distribute, sublicense, and/or sell copies
|
||||
* of the Software, and to permit persons to whom the Software is
|
||||
* furnished to do so, subject to the following conditions:
|
||||
*
|
||||
* The above copyright notice and this permission notice (including
|
||||
* the next paragraph) shall be included in all copies or substantial
|
||||
* portions of the Software.
|
||||
*
|
||||
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
||||
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
||||
* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
||||
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
|
||||
* HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
|
||||
* WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
||||
* DEALINGS IN THE SOFTWARE.
|
||||
*/
|
||||
|
||||
#include "libhsakmt.h"
|
||||
|
||||
// HSAKMT global data
|
||||
|
||||
int kfd_fd = -1;
|
||||
unsigned long dxg_open_count;
|
||||
unsigned long system_properties_count;
|
||||
pthread_mutex_t hsakmt_mutex = PTHREAD_MUTEX_INITIALIZER;
|
||||
bool is_dgpu;
|
||||
|
||||
#ifndef PAGE_SIZE
|
||||
int PAGE_SIZE;
|
||||
#endif
|
||||
|
||||
int PAGE_SHIFT;
|
||||
|
||||
/* whether to check all dGPUs in the topology support SVM API */
|
||||
bool is_svm_api_supported;
|
||||
/* zfb is mainly used during emulation */
|
||||
int zfb_support;
|
||||
/* handle vendor specific packet */
|
||||
int vendor_packet_support;
|
||||
+1064
Failā izmaiņas netiks attēlotas, jo tās ir par lielu
Ielādēt izmaiņas
@@ -0,0 +1,363 @@
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
//
|
||||
// The University of Illinois/NCSA
|
||||
// Open Source License (NCSA)
|
||||
//
|
||||
// Copyright (c) 2014-2020, Advanced Micro Devices, Inc. All rights reserved.
|
||||
//
|
||||
// Developed by:
|
||||
//
|
||||
// AMD Research and AMD HSA Software Development
|
||||
//
|
||||
// Advanced Micro Devices, Inc.
|
||||
//
|
||||
// www.amd.com
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
// of this software and associated documentation files (the "Software"), to
|
||||
// deal with the Software without restriction, including without limitation
|
||||
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
// and/or sell copies of the Software, and to permit persons to whom the
|
||||
// Software is furnished to do so, subject to the following conditions:
|
||||
//
|
||||
// - Redistributions of source code must retain the above copyright notice,
|
||||
// this list of conditions and the following disclaimers.
|
||||
// - Redistributions in binary form must reproduce the above copyright
|
||||
// notice, this list of conditions and the following disclaimers in
|
||||
// the documentation and/or other materials provided with the distribution.
|
||||
// - Neither the names of Advanced Micro Devices, Inc,
|
||||
// nor the names of its contributors may be used to endorse or promote
|
||||
// products derived from this Software without specific prior written
|
||||
// permission.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
|
||||
// THE CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
|
||||
// OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
|
||||
// ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
||||
// DEALINGS WITH THE SOFTWARE.
|
||||
//
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
// This file is used only for open source cmake builds, if we hardcode the
|
||||
// register values in amd_aql_queue.cpp then this file won't be required. For
|
||||
// now we are using this file where register details are spelled out in the
|
||||
// structs/unions below.
|
||||
#ifndef HSA_RUNTME_CORE_INC_REGISTERS_H_
|
||||
#define HSA_RUNTME_CORE_INC_REGISTERS_H_
|
||||
|
||||
typedef enum SQ_RSRC_BUF_TYPE {
|
||||
SQ_RSRC_BUF = 0x00000000,
|
||||
SQ_RSRC_BUF_RSVD_1 = 0x00000001,
|
||||
SQ_RSRC_BUF_RSVD_2 = 0x00000002,
|
||||
SQ_RSRC_BUF_RSVD_3 = 0x00000003,
|
||||
} SQ_RSRC_BUF_TYPE;
|
||||
|
||||
typedef enum BUF_DATA_FORMAT {
|
||||
BUF_DATA_FORMAT_INVALID = 0x00000000,
|
||||
BUF_DATA_FORMAT_8 = 0x00000001,
|
||||
BUF_DATA_FORMAT_16 = 0x00000002,
|
||||
BUF_DATA_FORMAT_8_8 = 0x00000003,
|
||||
BUF_DATA_FORMAT_32 = 0x00000004,
|
||||
BUF_DATA_FORMAT_16_16 = 0x00000005,
|
||||
BUF_DATA_FORMAT_10_11_11 = 0x00000006,
|
||||
BUF_DATA_FORMAT_11_11_10 = 0x00000007,
|
||||
BUF_DATA_FORMAT_10_10_10_2 = 0x00000008,
|
||||
BUF_DATA_FORMAT_2_10_10_10 = 0x00000009,
|
||||
BUF_DATA_FORMAT_8_8_8_8 = 0x0000000a,
|
||||
BUF_DATA_FORMAT_32_32 = 0x0000000b,
|
||||
BUF_DATA_FORMAT_16_16_16_16 = 0x0000000c,
|
||||
BUF_DATA_FORMAT_32_32_32 = 0x0000000d,
|
||||
BUF_DATA_FORMAT_32_32_32_32 = 0x0000000e,
|
||||
BUF_DATA_FORMAT_RESERVED_15 = 0x0000000f,
|
||||
} BUF_DATA_FORMAT;
|
||||
|
||||
typedef enum BUF_NUM_FORMAT {
|
||||
BUF_NUM_FORMAT_UNORM = 0x00000000,
|
||||
BUF_NUM_FORMAT_SNORM = 0x00000001,
|
||||
BUF_NUM_FORMAT_USCALED = 0x00000002,
|
||||
BUF_NUM_FORMAT_SSCALED = 0x00000003,
|
||||
BUF_NUM_FORMAT_UINT = 0x00000004,
|
||||
BUF_NUM_FORMAT_SINT = 0x00000005,
|
||||
BUF_NUM_FORMAT_SNORM_OGL__SI__CI = 0x00000006,
|
||||
BUF_NUM_FORMAT_RESERVED_6__VI = 0x00000006,
|
||||
BUF_NUM_FORMAT_FLOAT = 0x00000007,
|
||||
} BUF_NUM_FORMAT;
|
||||
|
||||
typedef enum BUF_FORMAT {
|
||||
BUF_FORMAT_32_UINT = 0x00000014,
|
||||
} BUF_FORMAT;
|
||||
|
||||
typedef enum SQ_SEL_XYZW01 {
|
||||
SQ_SEL_0 = 0x00000000,
|
||||
SQ_SEL_1 = 0x00000001,
|
||||
SQ_SEL_RESERVED_0 = 0x00000002,
|
||||
SQ_SEL_RESERVED_1 = 0x00000003,
|
||||
SQ_SEL_X = 0x00000004,
|
||||
SQ_SEL_Y = 0x00000005,
|
||||
SQ_SEL_Z = 0x00000006,
|
||||
SQ_SEL_W = 0x00000007,
|
||||
} SQ_SEL_XYZW01;
|
||||
|
||||
union COMPUTE_TMPRING_SIZE {
|
||||
struct {
|
||||
#if defined(LITTLEENDIAN_CPU)
|
||||
unsigned int WAVES : 12;
|
||||
unsigned int WAVESIZE : 13;
|
||||
unsigned int : 7;
|
||||
#elif defined(BIGENDIAN_CPU)
|
||||
unsigned int : 7;
|
||||
unsigned int WAVESIZE : 13;
|
||||
unsigned int WAVES : 12;
|
||||
#endif
|
||||
} bitfields, bits;
|
||||
unsigned int u32All;
|
||||
signed int i32All;
|
||||
float f32All;
|
||||
};
|
||||
|
||||
union COMPUTE_TMPRING_SIZE_GFX11 {
|
||||
struct {
|
||||
#if defined(LITTLEENDIAN_CPU)
|
||||
unsigned int WAVES : 12;
|
||||
unsigned int WAVESIZE : 15;
|
||||
unsigned int : 5;
|
||||
#elif defined(BIGENDIAN_CPU)
|
||||
unsigned int : 5;
|
||||
unsigned int WAVESIZE : 15;
|
||||
unsigned int WAVES : 12;
|
||||
#endif
|
||||
} bitfields, bits;
|
||||
unsigned int u32All;
|
||||
signed int i32All;
|
||||
float f32All;
|
||||
};
|
||||
|
||||
union COMPUTE_TMPRING_SIZE_GFX12 {
|
||||
struct {
|
||||
#if defined(LITTLEENDIAN_CPU)
|
||||
unsigned int WAVES : 12;
|
||||
unsigned int WAVESIZE : 18;
|
||||
unsigned int : 2;
|
||||
#elif defined(BIGENDIAN_CPU)
|
||||
unsigned int : 2;
|
||||
unsigned int WAVESIZE : 18;
|
||||
unsigned int WAVES : 12;
|
||||
#endif
|
||||
} bitfields, bits;
|
||||
unsigned int u32All;
|
||||
signed int i32All;
|
||||
float f32All;
|
||||
};
|
||||
|
||||
union SQ_BUF_RSRC_WORD0 {
|
||||
struct {
|
||||
#if defined(LITTLEENDIAN_CPU)
|
||||
unsigned int BASE_ADDRESS : 32;
|
||||
#elif defined(BIGENDIAN_CPU)
|
||||
unsigned int BASE_ADDRESS : 32;
|
||||
#endif
|
||||
} bitfields, bits;
|
||||
unsigned int u32All;
|
||||
signed int i32All;
|
||||
float f32All;
|
||||
};
|
||||
|
||||
|
||||
union SQ_BUF_RSRC_WORD1 {
|
||||
struct {
|
||||
#if defined(LITTLEENDIAN_CPU)
|
||||
unsigned int BASE_ADDRESS_HI : 16;
|
||||
unsigned int STRIDE : 14;
|
||||
unsigned int CACHE_SWIZZLE : 1;
|
||||
unsigned int SWIZZLE_ENABLE : 1;
|
||||
#elif defined(BIGENDIAN_CPU)
|
||||
unsigned int SWIZZLE_ENABLE : 1;
|
||||
unsigned int CACHE_SWIZZLE : 1;
|
||||
unsigned int STRIDE : 14;
|
||||
unsigned int BASE_ADDRESS_HI : 16;
|
||||
#endif
|
||||
} bitfields, bits;
|
||||
unsigned int u32All;
|
||||
signed int i32All;
|
||||
float f32All;
|
||||
};
|
||||
|
||||
union SQ_BUF_RSRC_WORD1_GFX11 {
|
||||
struct {
|
||||
#if defined(LITTLEENDIAN_CPU)
|
||||
unsigned int BASE_ADDRESS_HI : 16;
|
||||
unsigned int STRIDE : 14;
|
||||
unsigned int SWIZZLE_ENABLE : 2;
|
||||
#elif defined(BIGENDIAN_CPU)
|
||||
unsigned int SWIZZLE_ENABLE : 2;
|
||||
unsigned int STRIDE : 14;
|
||||
unsigned int BASE_ADDRESS_HI : 16;
|
||||
#endif
|
||||
} bitfields, bits;
|
||||
unsigned int u32All;
|
||||
signed int i32All;
|
||||
float f32All;
|
||||
};
|
||||
|
||||
|
||||
union SQ_BUF_RSRC_WORD2 {
|
||||
struct {
|
||||
#if defined(LITTLEENDIAN_CPU)
|
||||
unsigned int NUM_RECORDS : 32;
|
||||
#elif defined(BIGENDIAN_CPU)
|
||||
unsigned int NUM_RECORDS : 32;
|
||||
#endif
|
||||
} bitfields, bits;
|
||||
unsigned int u32All;
|
||||
signed int i32All;
|
||||
float f32All;
|
||||
};
|
||||
|
||||
|
||||
union SQ_BUF_RSRC_WORD3 {
|
||||
struct {
|
||||
#if defined(LITTLEENDIAN_CPU)
|
||||
unsigned int DST_SEL_X : 3;
|
||||
unsigned int DST_SEL_Y : 3;
|
||||
unsigned int DST_SEL_Z : 3;
|
||||
unsigned int DST_SEL_W : 3;
|
||||
unsigned int NUM_FORMAT : 3;
|
||||
unsigned int DATA_FORMAT : 4;
|
||||
unsigned int ELEMENT_SIZE : 2;
|
||||
unsigned int INDEX_STRIDE : 2;
|
||||
unsigned int ADD_TID_ENABLE : 1;
|
||||
unsigned int ATC__CI__VI : 1;
|
||||
unsigned int HASH_ENABLE : 1;
|
||||
unsigned int HEAP : 1;
|
||||
unsigned int MTYPE__CI__VI : 3;
|
||||
unsigned int TYPE : 2;
|
||||
#elif defined(BIGENDIAN_CPU)
|
||||
unsigned int TYPE : 2;
|
||||
unsigned int MTYPE__CI__VI : 3;
|
||||
unsigned int HEAP : 1;
|
||||
unsigned int HASH_ENABLE : 1;
|
||||
unsigned int ATC__CI__VI : 1;
|
||||
unsigned int ADD_TID_ENABLE : 1;
|
||||
unsigned int INDEX_STRIDE : 2;
|
||||
unsigned int ELEMENT_SIZE : 2;
|
||||
unsigned int DATA_FORMAT : 4;
|
||||
unsigned int NUM_FORMAT : 3;
|
||||
unsigned int DST_SEL_W : 3;
|
||||
unsigned int DST_SEL_Z : 3;
|
||||
unsigned int DST_SEL_Y : 3;
|
||||
unsigned int DST_SEL_X : 3;
|
||||
#endif
|
||||
} bitfields, bits;
|
||||
unsigned int u32All;
|
||||
signed int i32All;
|
||||
float f32All;
|
||||
};
|
||||
|
||||
union SQ_BUF_RSRC_WORD3_GFX10 {
|
||||
struct {
|
||||
#if defined(LITTLEENDIAN_CPU)
|
||||
unsigned int DST_SEL_X : 3;
|
||||
unsigned int DST_SEL_Y : 3;
|
||||
unsigned int DST_SEL_Z : 3;
|
||||
unsigned int DST_SEL_W : 3;
|
||||
unsigned int FORMAT : 7;
|
||||
unsigned int RESERVED1 : 2;
|
||||
unsigned int INDEX_STRIDE : 2;
|
||||
unsigned int ADD_TID_ENABLE : 1;
|
||||
unsigned int RESOURCE_LEVEL : 1;
|
||||
unsigned int RESERVED2 : 3;
|
||||
unsigned int OOB_SELECT : 2;
|
||||
unsigned int TYPE : 2;
|
||||
#elif defined(BIGENDIAN_CPU)
|
||||
unsigned int TYPE : 2;
|
||||
unsigned int OOB_SELECT : 2;
|
||||
unsigned int RESERVED2 : 3;
|
||||
unsigned int RESOURCE_LEVEL : 1;
|
||||
unsigned int ADD_TID_ENABLE : 1;
|
||||
unsigned int INDEX_STRIDE : 2;
|
||||
unsigned int RESERVED1 : 2;
|
||||
unsigned int FORMAT : 7;
|
||||
unsigned int DST_SEL_W : 3;
|
||||
unsigned int DST_SEL_Z : 3;
|
||||
unsigned int DST_SEL_Y : 3;
|
||||
unsigned int DST_SEL_X : 3;
|
||||
#endif
|
||||
} bitfields, bits;
|
||||
unsigned int u32All;
|
||||
signed int i32All;
|
||||
float f32All;
|
||||
};
|
||||
|
||||
// From V# Table
|
||||
union SQ_BUF_RSRC_WORD3_GFX11 {
|
||||
struct {
|
||||
#if defined(LITTLEENDIAN_CPU)
|
||||
unsigned int DST_SEL_X : 3;
|
||||
unsigned int DST_SEL_Y : 3;
|
||||
unsigned int DST_SEL_Z : 3;
|
||||
unsigned int DST_SEL_W : 3;
|
||||
unsigned int FORMAT : 6;
|
||||
unsigned int RESERVED1 : 3;
|
||||
unsigned int INDEX_STRIDE : 2;
|
||||
unsigned int ADD_TID_ENABLE : 1;
|
||||
unsigned int RESERVED2 : 4;
|
||||
unsigned int OOB_SELECT : 2;
|
||||
unsigned int TYPE : 2;
|
||||
#elif defined(BIGENDIAN_CPU)
|
||||
unsigned int TYPE : 2;
|
||||
unsigned int OOB_SELECT : 2;
|
||||
unsigned int RESERVED2 : 4;
|
||||
unsigned int ADD_TID_ENABLE : 1;
|
||||
unsigned int INDEX_STRIDE : 2;
|
||||
unsigned int RESERVED1 : 3;
|
||||
unsigned int FORMAT : 6;
|
||||
unsigned int DST_SEL_W : 3;
|
||||
unsigned int DST_SEL_Z : 3;
|
||||
unsigned int DST_SEL_Y : 3;
|
||||
unsigned int DST_SEL_X : 3;
|
||||
#endif
|
||||
} bitfields, bits;
|
||||
unsigned int u32All;
|
||||
signed int i32All;
|
||||
float f32All;
|
||||
};
|
||||
// From V# Table
|
||||
union SQ_BUF_RSRC_WORD3_GFX12 {
|
||||
struct {
|
||||
#if defined(LITTLEENDIAN_CPU)
|
||||
unsigned int DST_SEL_X : 3;
|
||||
unsigned int DST_SEL_Y : 3;
|
||||
unsigned int DST_SEL_Z : 3;
|
||||
unsigned int DST_SEL_W : 3;
|
||||
unsigned int FORMAT : 6;
|
||||
unsigned int RESERVED1 : 3;
|
||||
unsigned int INDEX_STRIDE : 2;
|
||||
unsigned int ADD_TID_ENABLE : 1;
|
||||
unsigned int WRITE_COMPRESS_ENABLE : 1;
|
||||
unsigned int COMPRESSION_EN : 1;
|
||||
unsigned int COMPRESSION_ACCESS_MODE : 2;
|
||||
unsigned int OOB_SELECT : 2;
|
||||
unsigned int TYPE : 2;
|
||||
#elif defined(BIGENDIAN_CPU)
|
||||
unsigned int TYPE : 2;
|
||||
unsigned int OOB_SELECT : 2;
|
||||
unsigned int COMPRESSION_ACCESS_MODE : 2;
|
||||
unsigned int COMPRESSION_EN : 1;
|
||||
unsigned int WRITE_COMPRESS_ENABLE : 1;
|
||||
unsigned int ADD_TID_ENABLE : 1;
|
||||
unsigned int INDEX_STRIDE : 2;
|
||||
unsigned int RESERVED1 : 3;
|
||||
unsigned int FORMAT : 6;
|
||||
unsigned int DST_SEL_W : 3;
|
||||
unsigned int DST_SEL_Z : 3;
|
||||
unsigned int DST_SEL_Y : 3;
|
||||
unsigned int DST_SEL_X : 3;
|
||||
#endif
|
||||
} bitfields, bits;
|
||||
unsigned int u32All;
|
||||
signed int i32All;
|
||||
float f32All;
|
||||
};
|
||||
#endif // header guard
|
||||
@@ -0,0 +1,131 @@
|
||||
#ifndef _ROCR_PROXY_H_
|
||||
#define _ROCR_PROXY_H_
|
||||
|
||||
#include <vector>
|
||||
|
||||
namespace rocr_proxy {
|
||||
enum AllocDomain {
|
||||
kSystem,
|
||||
kLocal,
|
||||
kUserMemory,
|
||||
kUserQueue,
|
||||
kDomainCount,
|
||||
};
|
||||
|
||||
enum MemFlag {
|
||||
kFineGrain = (1ULL << 0),
|
||||
kKernarg = (1ULL << 1),
|
||||
};
|
||||
|
||||
enum EngineFlag {
|
||||
KCOMPUTE0 = (1ULL << 0),
|
||||
KDRMDMA = (1ULL << 1),
|
||||
KDRMDMA1 = (1ULL << 2),
|
||||
};
|
||||
|
||||
enum SchedLevel {
|
||||
kLow = 0,
|
||||
kNormal = 1,
|
||||
kHigh = 2,
|
||||
};
|
||||
|
||||
enum AsicFamilyType {
|
||||
kVega10,
|
||||
kNavi10,
|
||||
kSiennaCichlid,
|
||||
kPlumBONITO,
|
||||
kNavi44,
|
||||
kNavi48
|
||||
};
|
||||
|
||||
struct HwsInfo {
|
||||
union {
|
||||
struct {
|
||||
uint32_t gfxHwsEnabled : 1;
|
||||
uint32_t computeHwsEnabled : 1;
|
||||
uint32_t dmaHwsEnabled : 1;
|
||||
uint32_t dma1HwsEnabled : 1;
|
||||
uint32_t reserved : 28;
|
||||
} hwsMask;
|
||||
uint32_t osHwsEnableFlags;
|
||||
};
|
||||
uint64_t engineOrdinalMask; // Indicates which engines (by ordinal) support MES HWS
|
||||
};
|
||||
|
||||
typedef struct {
|
||||
int major;
|
||||
int minor;
|
||||
int stepping;
|
||||
bool is_dgpu;
|
||||
char product_name[MAX_PATH];
|
||||
const char *uuid;
|
||||
AsicFamilyType family;
|
||||
uint32_t device_id;
|
||||
uint32_t wavefront_size;
|
||||
uint32_t compute_unit_count;
|
||||
uint32_t max_engine_clock_mhz;
|
||||
uint32_t watch_points_num;
|
||||
uint32_t pci_bus_addr;
|
||||
uint32_t memory_bus_width;
|
||||
uint32_t max_memory_clock_mhz;
|
||||
uint64_t gpu_counter_frequency;
|
||||
uint32_t wave_per_cu;
|
||||
uint32_t simd_per_cu;
|
||||
uint32_t max_scratch_slots_per_cu;
|
||||
uint32_t num_shader_engine;
|
||||
uint32_t shader_array_per_shader_engine;
|
||||
uint32_t domain;
|
||||
uint32_t num_gws;
|
||||
uint32_t asic_revision;
|
||||
uint64_t local_visible_heap_size;
|
||||
uint64_t local_invisible_heap_size;
|
||||
uint64_t private_aperture_base;
|
||||
uint64_t private_aperture_size;
|
||||
uint64_t shared_aperture_base;
|
||||
uint64_t shared_aperture_size;
|
||||
uint32_t user_queue_size;
|
||||
uint32_t lds_size;
|
||||
uint32_t big_page_alignment_size;
|
||||
uint32_t hw_big_page_min_alignment_size;
|
||||
uint32_t hw_big_page_alignment_size;
|
||||
bool enable_big_page_alignment;
|
||||
uint32_t mec_fw_version;
|
||||
uint32_t sdma_fw_version;
|
||||
uint32_t l1_cache_size;
|
||||
uint32_t l2_cache_size;
|
||||
uint32_t l3_cache_size;
|
||||
uint32_t gl2_cacheline_size;
|
||||
uint32_t num_cp_queues;
|
||||
HwsInfo hwsInfo;
|
||||
std::vector<int> sdma_schedid;
|
||||
uint32_t compute_schedid;
|
||||
bool state_shadowing_by_cpfw;
|
||||
bool platform_atomic_support;
|
||||
void *adapter_info;
|
||||
void *adapter_ex_info;
|
||||
} DeviceInfo;
|
||||
|
||||
int EngineOrdinal(int engine, DeviceInfo *device_info);
|
||||
bool GetHwsEnabled(int engine, DeviceInfo *device_info);
|
||||
bool ShouldDisableGpuTimeout(int engine, DeviceInfo *device_info);
|
||||
bool ParseAdapterInfo(D3DKMT_HANDLE adapter, DeviceInfo *device_info);
|
||||
bool QueryAdapterSupported(D3DKMT_HANDLE adapter);
|
||||
|
||||
uint32_t QueueEngine2EngineFlag(uint32_t queue_engine);
|
||||
void SetAllocationInfo(void *data, uint64_t size, AllocDomain domain,
|
||||
uint64_t addr, uint32_t mem_flags, uint32_t engine_flag, const DeviceInfo &device_info);
|
||||
bool CreatePrivateAllocInfo(int num_handles, void **ppdrv_priv, void **ppalloc_priv,
|
||||
int *pdrv_priv_data_size, int *palloc_priv_data_size);
|
||||
void DestroyPrivateAllocInfo(void *drv_priv, void *alloc_priv);
|
||||
|
||||
int CreateSubmitPrivData(void **priv_data, D3DKMT_HANDLE queue, uint64_t command_addr,
|
||||
uint64_t command_size, bool is_hw_queue);
|
||||
int CreateHwQueuePrivData(void **priv_data, D3DKMT_HANDLE context,
|
||||
bool FwManagedGfxState, SchedLevel level = kNormal);
|
||||
int CreateContextPrivData(void **priv_data, bool FwManagedGfxState);
|
||||
int CreatePowerOptPrivData(void **priv_data, bool restore);
|
||||
int CreateCalibratedTimestampsPrivData(void **priv_data);
|
||||
void QueryCalibratedTimestamps(void* priv, uint64_t* gpu, uint64_t* cpu);
|
||||
void DestroyPrivData(void *priv_data);
|
||||
}
|
||||
#endif
|
||||
@@ -0,0 +1,155 @@
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
//
|
||||
// The University of Illinois/NCSA
|
||||
// Open Source License (NCSA)
|
||||
//
|
||||
// Copyright (c) 2014-2015, Advanced Micro Devices, Inc. All rights reserved.
|
||||
//
|
||||
// Developed by:
|
||||
//
|
||||
// AMD Research and AMD HSA Software Development
|
||||
//
|
||||
// Advanced Micro Devices, Inc.
|
||||
//
|
||||
// www.amd.com
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
// of this software and associated documentation files (the "Software"), to
|
||||
// deal with the Software without restriction, including without limitation
|
||||
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
// and/or sell copies of the Software, and to permit persons to whom the
|
||||
// Software is furnished to do so, subject to the following conditions:
|
||||
//
|
||||
// - Redistributions of source code must retain the above copyright notice,
|
||||
// this list of conditions and the following disclaimers.
|
||||
// - Redistributions in binary form must reproduce the above copyright
|
||||
// notice, this list of conditions and the following disclaimers in
|
||||
// the documentation and/or other materials provided with the distribution.
|
||||
// - Neither the names of Advanced Micro Devices, Inc,
|
||||
// nor the names of its contributors may be used to endorse or promote
|
||||
// products derived from this Software without specific prior written
|
||||
// permission.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
|
||||
// THE CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
|
||||
// OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
|
||||
// ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
||||
// DEALINGS WITH THE SOFTWARE.
|
||||
//
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
#ifndef _ROCR_WDDM_TYPES_H_
|
||||
#define _ROCR_WDDM_TYPES_H_
|
||||
|
||||
#include <stdint.h>
|
||||
|
||||
#include <no_sal2.h>
|
||||
|
||||
typedef uint32_t UINT, *UINT_PTR;
|
||||
typedef int32_t INT32;
|
||||
typedef int32_t LONG;
|
||||
typedef uint32_t ULONG, *ULONG_PTR;
|
||||
typedef int64_t LONGLONG;
|
||||
typedef int64_t LONG64;
|
||||
typedef uint64_t ULONGLONG;
|
||||
typedef uint64_t ULONG64, *ULONG64_PTR;
|
||||
typedef uint8_t BYTE;
|
||||
typedef uint16_t WORD;
|
||||
typedef uint32_t DWORD;
|
||||
typedef int32_t BOOL;
|
||||
typedef int32_t NTSTATUS;
|
||||
typedef uint16_t USHORT;
|
||||
typedef uint16_t UINT16;
|
||||
typedef uint32_t UINT32;
|
||||
typedef uint64_t UINT64;
|
||||
typedef int32_t INT;
|
||||
typedef uint64_t SIZE_T;
|
||||
typedef void VOID;
|
||||
typedef float FLOAT;
|
||||
typedef char CHAR;
|
||||
typedef unsigned char UCHAR;
|
||||
typedef UCHAR BOOLEAN;
|
||||
typedef int16_t WCHAR;
|
||||
typedef void *HANDLE;
|
||||
typedef void *PVOID;
|
||||
typedef void *LPVOID;
|
||||
typedef const int16_t *PCWSTR;
|
||||
|
||||
#define ULONG ULONG
|
||||
#define ULONG_PTR ULONG_PTR
|
||||
#define USHORT USHORT
|
||||
|
||||
#define DECLARE_HANDLE(name) struct name##__{int unused;}; typedef struct name##__ *name
|
||||
#define C_ASSERT(e) typedef char __C_ASSERT__[(e)?1:-1]
|
||||
|
||||
DECLARE_HANDLE(HWND);
|
||||
DECLARE_HANDLE(HDC);
|
||||
DECLARE_HANDLE(PALETTEENTRY);
|
||||
|
||||
typedef struct tagPOINT {
|
||||
LONG x;
|
||||
LONG y;
|
||||
} POINT;
|
||||
|
||||
typedef struct tagRECT {
|
||||
LONG left;
|
||||
LONG top;
|
||||
LONG right;
|
||||
LONG bottom;
|
||||
} RECT;
|
||||
|
||||
typedef struct tagRECTL {
|
||||
LONG left;
|
||||
LONG top;
|
||||
LONG right;
|
||||
LONG bottom;
|
||||
} RECTL;
|
||||
|
||||
typedef union _LARGE_INTEGER {
|
||||
struct {
|
||||
DWORD LowPart;
|
||||
DWORD HighPart;
|
||||
} u;
|
||||
LONGLONG QuadPart;
|
||||
} LARGE_INTEGER;
|
||||
|
||||
typedef LARGE_INTEGER *PLARGE_INTEGER;
|
||||
|
||||
typedef struct _LUID {
|
||||
ULONG LowPart;
|
||||
LONG HighPart;
|
||||
} LUID, *PLUID;
|
||||
|
||||
typedef enum _DEVICE_POWER_STATE {
|
||||
PowerDeviceUnspecified = 0,
|
||||
PowerDeviceD0,
|
||||
PowerDeviceD1,
|
||||
PowerDeviceD2,
|
||||
PowerDeviceD3,
|
||||
PowerDeviceMaximum
|
||||
} DEVICE_POWER_STATE, *PDEVICE_POWER_STATE;
|
||||
|
||||
#define _Check_return_
|
||||
#define APIENTRY
|
||||
#define CONST const
|
||||
#define IN
|
||||
#define OUT
|
||||
#define FAR
|
||||
#define MAX_PATH 260
|
||||
#define __stdcall
|
||||
|
||||
#ifndef GUID_DEFINED
|
||||
#define GUID_DEFINED
|
||||
typedef struct _GUID {
|
||||
uint32_t Data1;
|
||||
uint16_t Data2;
|
||||
uint16_t Data3;
|
||||
uint8_t Data4[ 8 ];
|
||||
} GUID;
|
||||
#endif
|
||||
|
||||
#include <guiddef.h>
|
||||
|
||||
#endif
|
||||
@@ -0,0 +1,77 @@
|
||||
/* Copyright (c) 2023 Advanced Micro Devices, Inc. All rights reserved. */
|
||||
|
||||
#ifndef _CMD_UTIL_H_
|
||||
#define _CMD_UTIL_H_
|
||||
|
||||
#include <string.h>
|
||||
#include "hsa-runtime/inc/hsa.h"
|
||||
#include "hsa-runtime/inc/amd_hsa_queue.h"
|
||||
#include "hsa-runtime/inc/amd_hsa_kernel_code.h"
|
||||
#include "inc/pm4_cmds.h"
|
||||
#include "util/utils.h"
|
||||
|
||||
namespace rocr {
|
||||
namespace core {
|
||||
|
||||
struct DispatchInfo {
|
||||
uint8_t major;
|
||||
hsa_kernel_dispatch_packet_t *pPacket;
|
||||
void *pEntry;
|
||||
const amd_kernel_code_t *pKernelObject;
|
||||
uint32_t ldsBlks;
|
||||
amd_queue_t *pAmdQueue;
|
||||
bool wave32;
|
||||
uint32_t srd;
|
||||
void *pScratchBase;
|
||||
uint32_t scratchSizePerWave;
|
||||
uint32_t scratchBaseOffset[2];
|
||||
uint32_t offsetCnt;
|
||||
};
|
||||
|
||||
class CmdUtil {
|
||||
public:
|
||||
CmdUtil() {};
|
||||
~CmdUtil() {};
|
||||
|
||||
size_t BuildCopyData(
|
||||
uint64_t *pDstAddr,
|
||||
void *pBuffer,
|
||||
uint32_t dstSel = dst_sel__mec_copy_data__tc_l2,
|
||||
uint32_t dstCachePolicy = dst_cache_policy__mec_copy_data__stream,
|
||||
uint32_t srcSel = src_sel__mec_copy_data__gpu_clock_count,
|
||||
uint32_t srcCachePolicy = src_cache_policy__mec_copy_data__lru,
|
||||
uint32_t countSel = count_sel__mec_copy_data__64_bits_of_data,
|
||||
uint32_t wrConfirm = wr_confirm__mec_copy_data__wait_for_confirmation);
|
||||
|
||||
size_t BuildBarrier(
|
||||
void *pBuffer,
|
||||
uint32_t eventIndex = event_index__mec_event_write__cs_partial_flush,
|
||||
uint32_t eventType = CS_PARTIAL_FLUSH);
|
||||
|
||||
size_t BuildAcquireMem(
|
||||
uint8_t major,
|
||||
void *pBuffer);
|
||||
|
||||
size_t BuildScratch(
|
||||
void *pScratchBase,
|
||||
void *pBuffer);
|
||||
|
||||
size_t BuildComputeShaderParams(
|
||||
void *pBuffer);
|
||||
|
||||
size_t BuildDispatch(
|
||||
struct DispatchInfo *pInfo,
|
||||
void *pBuffer);
|
||||
|
||||
size_t BuildAtomicMem(
|
||||
uint64_t *pAddr,
|
||||
uint32_t atomic,
|
||||
void *pBuffer,
|
||||
uint32_t cachePolicy = cache_policy__mec_atomic_mem__stream,
|
||||
uint64_t srcData = 1);
|
||||
};
|
||||
|
||||
} // namespace core
|
||||
} // namespace rocr
|
||||
|
||||
#endif
|
||||
@@ -0,0 +1,245 @@
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
//
|
||||
// The University of Illinois/NCSA
|
||||
// Open Source License (NCSA)
|
||||
//
|
||||
// Copyright (c) 2020, Advanced Micro Devices, Inc. All rights reserved.
|
||||
//
|
||||
// Developed by:
|
||||
//
|
||||
// AMD Research and AMD HSA Software Development
|
||||
//
|
||||
// Advanced Micro Devices, Inc.
|
||||
//
|
||||
// www.amd.com
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
// of this software and associated documentation files (the "Software"), to
|
||||
// deal with the Software without restriction, including without limitation
|
||||
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
// and/or sell copies of the Software, and to permit persons to whom the
|
||||
// Software is furnished to do so, subject to the following conditions:
|
||||
//
|
||||
// - Redistributions of source code must retain the above copyright notice,
|
||||
// this list of conditions and the following disclaimers.
|
||||
// - Redistributions in binary form must reproduce the above copyright
|
||||
// notice, this list of conditions and the following disclaimers in
|
||||
// the documentation and/or other materials provided with the distribution.
|
||||
// - Neither the names of Advanced Micro Devices, Inc,
|
||||
// nor the names of its contributors may be used to endorse or promote
|
||||
// products derived from this Software without specific prior written
|
||||
// permission.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
|
||||
// THE CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
|
||||
// OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
|
||||
// ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
||||
// DEALINGS WITH THE SOFTWARE.
|
||||
//
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
#ifndef _ROCR_WDDM_H_
|
||||
#define _ROCR_WDDM_H_
|
||||
|
||||
#include <cassert>
|
||||
#include <ntstatus.h>
|
||||
|
||||
#include <atomic>
|
||||
#include <memory>
|
||||
#include <vector>
|
||||
|
||||
#include "inc/wddm/types.h"
|
||||
#include "inc/rocr_proxy/rocr_proxy.h"
|
||||
#include "inc/wddm/va_mgr.h"
|
||||
#include "inc/wddm/status.h"
|
||||
#include "inc/wddm/types.h"
|
||||
#include "inc/wddm/gpu_memory.h"
|
||||
#include "inc/wddm/cmd_util.h"
|
||||
|
||||
namespace rocr {
|
||||
namespace core {
|
||||
|
||||
//class Queue;
|
||||
class WDDMQueue;
|
||||
|
||||
// WSL2 hyperv GPADL protocol limitation
|
||||
#define MAX_USERPTR_BLOCK_SIZE 0xf0000000
|
||||
|
||||
class WDDMDevice {
|
||||
public:
|
||||
static constexpr size_t GpuMemoryChunkSize = 2 * (1ULL << 30); // 2 GB
|
||||
|
||||
WDDMDevice(D3DKMT_HANDLE adapter, LUID adapter_luid);
|
||||
~WDDMDevice();
|
||||
|
||||
int Major() { return device_info_.major; }
|
||||
int Minor() { return device_info_.minor; }
|
||||
int Stepping() { return device_info_.stepping; }
|
||||
bool IsDgpu() { return device_info_.is_dgpu; }
|
||||
const char *ProductName() { return device_info_.product_name; }
|
||||
const char *Uuid() { return device_info_.uuid; }
|
||||
rocr_proxy::AsicFamilyType GfxFamily() { return device_info_.family; }
|
||||
uint32_t DeviceId() { return device_info_.device_id; }
|
||||
uint32_t WavefrontSize() { return device_info_.wavefront_size; }
|
||||
uint32_t ComputeUnitCount() { return device_info_.compute_unit_count; }
|
||||
uint32_t MaxEngineClockMhz() { return device_info_.max_engine_clock_mhz; }
|
||||
uint32_t WatchPointsNum() { return device_info_.watch_points_num; }
|
||||
uint32_t PciBusAddr() { return device_info_.pci_bus_addr; }
|
||||
|
||||
uint32_t MemoryBusWidth() { return device_info_.memory_bus_width; }
|
||||
uint32_t MaxMemoryClockMhz() { return device_info_.max_memory_clock_mhz; }
|
||||
uint32_t WavePerCu() { return device_info_.wave_per_cu; }
|
||||
uint32_t SimdPerCu() { return device_info_.simd_per_cu; }
|
||||
uint32_t MaxScratchSlotsPerCu() { return device_info_.max_scratch_slots_per_cu; }
|
||||
uint32_t NumShaderEngine() { return device_info_.num_shader_engine; }
|
||||
uint32_t ShaderArrayPerShaderEngine() { return device_info_.shader_array_per_shader_engine; }
|
||||
uint32_t NumSdmaEngine() { return device_info_.sdma_schedid.size(); }
|
||||
uint32_t Domain() { return device_info_.domain; }
|
||||
uint32_t NumGws() { return device_info_.num_gws; }
|
||||
uint32_t AsicRevision() { return device_info_.asic_revision; }
|
||||
uint64_t LocalHeapSize() { return device_info_.local_visible_heap_size + device_info_.local_invisible_heap_size; }
|
||||
uint64_t LocalVisibleHeapSize() { return device_info_.local_visible_heap_size; }
|
||||
uint64_t LocalInvisibleHeapSize() { return device_info_.local_invisible_heap_size; }
|
||||
uint64_t PrivateApertureBase() { return device_info_.private_aperture_base; }
|
||||
uint64_t PrivateApertureSize() { return device_info_.private_aperture_size; }
|
||||
uint64_t SharedApertureBase() { return device_info_.shared_aperture_base; }
|
||||
uint64_t SharedApertureSize() { return device_info_.shared_aperture_size; }
|
||||
uint32_t LdsSize() { return device_info_.lds_size; }
|
||||
uint64_t GPUCounterFrequency() { return device_info_.gpu_counter_frequency; }
|
||||
uint32_t GetSwsQueueSize(void) const { return device_info_.user_queue_size; }
|
||||
uint32_t GetMecFwVersion() { return device_info_.mec_fw_version; }
|
||||
uint32_t GetSdmaFwVersion() { return device_info_.sdma_fw_version; }
|
||||
uint32_t GetL1CacheSize() { return device_info_.l1_cache_size; }
|
||||
uint32_t GetL2CacheSize() { return device_info_.l2_cache_size; }
|
||||
uint32_t GetL3CacheSize() { return device_info_.l3_cache_size; }
|
||||
uint32_t Gl2CacheLineSize() { return device_info_.gl2_cacheline_size; }
|
||||
bool SupportStateShadowingByCpFw(void) const { return device_info_.state_shadowing_by_cpfw; }
|
||||
bool SupportPlatformAtomic(void) const { return device_info_.platform_atomic_support; }
|
||||
uint32_t GetSdmaEngine(uint32_t idx) {
|
||||
assert(idx < NumSdmaEngine());
|
||||
return device_info_.sdma_schedid[idx];
|
||||
}
|
||||
uint32_t GetComputeEngine() { return device_info_.compute_schedid; }
|
||||
|
||||
uint64_t VramAvail();
|
||||
|
||||
void GetClockCounters(uint64_t *gpu, uint64_t *cpu);
|
||||
uint32_t GetNumCpQueues() { return device_info_.num_cp_queues; }
|
||||
|
||||
bool CreateSyncobj(D3DKMT_HANDLE *handle, uint64_t **addr);
|
||||
void DestroySyncobj(D3DKMT_HANDLE handle);
|
||||
|
||||
bool CreateQueue(WDDMQueue *queue);
|
||||
void DestroyQueue(WDDMQueue *queue);
|
||||
bool CreateHwQueue(WDDMQueue *queue);
|
||||
bool DestroyHwQueue(WDDMQueue *queue);
|
||||
bool SubmitToSwQueue(WDDMQueue *queue, uint64_t command_addr,
|
||||
uint64_t command_size, uint64_t fence_value);
|
||||
bool SubmitToHwQueue(WDDMQueue *queue, uint64_t command_addr,
|
||||
uint64_t command_size, uint64_t fence_value);
|
||||
|
||||
bool WaitPagingFence(WDDMQueue *queue) {
|
||||
uint64_t value = page_fence_value_;
|
||||
|
||||
if (*page_fence_addr_ < value &&
|
||||
!GpuWait(queue, &page_syncobj_, &value, 1))
|
||||
return false;
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
bool GpuWait(WDDMQueue *queue, const D3DKMT_HANDLE *syncobjs,
|
||||
uint64_t *values, int count);
|
||||
bool GpuSignal(D3DKMT_HANDLE context, const D3DKMT_HANDLE *syncobjs,
|
||||
uint64_t *value, int count);
|
||||
bool CpuWait(const D3DKMT_HANDLE *syncobjs, uint64_t *value,
|
||||
int count, bool wait_any);
|
||||
bool WaitOnPagingFenceFromCpu();
|
||||
|
||||
uint32_t LdsBlocks(const hsa_kernel_dispatch_packet_t *pkt);
|
||||
uint32_t GetCmdbufSize(void) const { return cmdbuf_size_; }
|
||||
uint32_t GetAqlFrameSize(void) const { return cmdbuf_aql_frame_size_; }
|
||||
static uint32_t GetAqlFrameNum(void) { return cmdbuf_aql_frame_num_; }
|
||||
|
||||
// Both legacy HWS and stage 1 HWS use KMD to alloc use queue memory,
|
||||
// return false by default
|
||||
bool AllocUserQueueMemFromUMD(void) const { return false; }
|
||||
|
||||
bool IsHwsEnabled(int engine) {
|
||||
return rocr_proxy::GetHwsEnabled(engine, &device_info_);
|
||||
}
|
||||
|
||||
void UpdatePageFence(uint64_t fence_value);
|
||||
|
||||
D3DKMT_HANDLE PagingQueue() const { return page_queue_; }
|
||||
D3DKMT_HANDLE PagingFence() const { return page_syncobj_; }
|
||||
D3DKMT_HANDLE DeviceHandle() const { return device_; }
|
||||
LUID GetLuid() const { return adapter_luid_; }
|
||||
|
||||
const rocr_proxy::DeviceInfo& DeviceInfo() const { return device_info_; }
|
||||
|
||||
ErrorCode ReserveGpuVirtualAddress(rocr_proxy::AllocDomain domain,
|
||||
gpusize hit_base_addr,
|
||||
gpusize size,
|
||||
gpusize *out_gpu_virtual_addr,
|
||||
gpusize alignment,
|
||||
bool lock=false);
|
||||
|
||||
ErrorCode FreeGpuVirtualAddress(rocr_proxy::AllocDomain domain,
|
||||
gpusize base_addr,
|
||||
gpusize size);
|
||||
|
||||
ErrorCode CreateGpuMemory(const GpuMemoryCreateInfo &create_info, GpuMemory **gpu_mem);
|
||||
private:
|
||||
bool ParseDeviceInfo(void);
|
||||
void DestroyDeviceInfo(void);
|
||||
bool CreateDevice(void);
|
||||
bool DestroyDevice(void);
|
||||
bool CreatePagingQueue(void);
|
||||
bool DestroyPagingQueue(void);
|
||||
void *Lock(D3DKMT_HANDLE handle);
|
||||
bool Unlock(D3DKMT_HANDLE handle);
|
||||
bool CreateContext(int engine, D3DKMT_HANDLE *handle);
|
||||
bool DestroyContext(D3DKMT_HANDLE handle);
|
||||
|
||||
void SetPowerOptimization(bool restore);
|
||||
void InitCmdbufInfo(void);
|
||||
bool ReserveSystemHeapSpace(void);
|
||||
bool FreeSystemHeapSpace(void);
|
||||
bool ReserveLocalHeapSpace(void);
|
||||
bool CommitSystemHeapSpace(void* addr, int64_t size, bool lock=false);
|
||||
bool DecommitSystemHeapSpace(void* addr, int64_t size);
|
||||
bool FreeLocalHeapSpace(void);
|
||||
void InitVaMgr();
|
||||
|
||||
D3DKMT_HANDLE adapter_;
|
||||
LUID adapter_luid_;
|
||||
D3DKMT_HANDLE device_;
|
||||
|
||||
D3DKMT_HANDLE page_queue_;
|
||||
D3DKMT_HANDLE page_syncobj_;
|
||||
uint64_t *page_fence_addr_;
|
||||
std::atomic<uint64_t> page_fence_value_;
|
||||
|
||||
uint64_t local_heap_space_start_;
|
||||
uint64_t local_heap_space_size_;
|
||||
uint64_t system_heap_space_start_;
|
||||
uint64_t system_heap_space_size_;
|
||||
uint32_t cmdbuf_size_;
|
||||
uint32_t cmdbuf_aql_frame_size_;
|
||||
static const uint32_t cmdbuf_aql_frame_num_;
|
||||
// device info
|
||||
rocr_proxy::DeviceInfo device_info_;
|
||||
|
||||
std::unique_ptr<VaMgr> local_va_mgr_;
|
||||
//CmdUtil cmd_util;
|
||||
};
|
||||
|
||||
NTSTATUS WDDMGetAdapters(D3DKMT_ADAPTERINFO *&adapters, int &num_adapters);
|
||||
|
||||
} // namespace core
|
||||
} // namespace rocr
|
||||
|
||||
#endif
|
||||
@@ -0,0 +1,218 @@
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
//
|
||||
// The University of Illinois/NCSA
|
||||
// Open Source License (NCSA)
|
||||
//
|
||||
// Copyright (c) 2020, Advanced Micro Devices, Inc. All rights reserved.
|
||||
//
|
||||
// Developed by:
|
||||
//
|
||||
// AMD Research and AMD HSA Software Development
|
||||
//
|
||||
// Advanced Micro Devices, Inc.
|
||||
//
|
||||
// www.amd.com
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
// of this software and associated documentation files (the "Software"), to
|
||||
// deal with the Software without restriction, including without limitation
|
||||
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
// and/or sell copies of the Software, and to permit persons to whom the
|
||||
// Software is furnished to do so, subject to the following conditions:
|
||||
//
|
||||
// - Redistributions of source code must retain the above copyright notice,
|
||||
// this list of conditions and the following disclaimers.
|
||||
// - Redistributions in binary form must reproduce the above copyright
|
||||
// notice, this list of conditions and the following disclaimers in
|
||||
// the documentation and/or other materials provided with the distribution.
|
||||
// - Neither the names of Advanced Micro Devices, Inc,
|
||||
// nor the names of its contributors may be used to endorse or promote
|
||||
// products derived from this Software without specific prior written
|
||||
// permission.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
|
||||
// THE CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
|
||||
// OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
|
||||
// ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
||||
// DEALINGS WITH THE SOFTWARE.
|
||||
//
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
#ifndef _ROCR_GPU_MEMORY_H_
|
||||
#define _ROCR_GPU_MEMORY_H_
|
||||
|
||||
#include <cstddef>
|
||||
#include <cstdint>
|
||||
#include "util/utils.h"
|
||||
#include "inc/wddm/types.h"
|
||||
#include "inc/wddm/thunks.h"
|
||||
#include "inc/rocr_proxy/rocr_proxy.h"
|
||||
|
||||
namespace rocr {
|
||||
namespace core {
|
||||
|
||||
class WDDMDevice;
|
||||
|
||||
union GpuMemoryCreateFlags {
|
||||
struct {
|
||||
uint64_t virtual_alloc : 1;
|
||||
uint64_t physical_only : 1;
|
||||
uint64_t interprocess : 1;
|
||||
uint64_t locked : 1;
|
||||
uint64_t unused : 60;
|
||||
};
|
||||
uint64_t reserved;
|
||||
};
|
||||
|
||||
struct GpuMemoryCreateInfo {
|
||||
GpuMemoryCreateInfo() {
|
||||
flags.reserved = 0;
|
||||
domain = rocr_proxy::kLocal;
|
||||
size = 0;
|
||||
alignment = 0;
|
||||
mem_flags = 0;
|
||||
engine_flag = 0;
|
||||
va_hint = 0;
|
||||
user_ptr = nullptr;
|
||||
dmabuf_fd = -1;
|
||||
}
|
||||
|
||||
GpuMemoryCreateFlags flags;
|
||||
rocr_proxy::AllocDomain domain;
|
||||
gpusize size;
|
||||
gpusize alignment;
|
||||
int mem_flags;
|
||||
int engine_flag;
|
||||
int dmabuf_fd; // Import from dmabuf
|
||||
|
||||
void *user_ptr;
|
||||
gpusize va_hint;
|
||||
};
|
||||
|
||||
struct GpuMemoryDesc {
|
||||
GpuMemoryDesc() {
|
||||
gpu_addr = 0;
|
||||
cpu_addr = nullptr;
|
||||
client_size = 0;
|
||||
size = alignment = 0;
|
||||
flags.reserved = 0;
|
||||
mem_flags = 0;
|
||||
engine_flag = 0;
|
||||
}
|
||||
|
||||
rocr_proxy::AllocDomain domain;
|
||||
LUID adapter_luid; // Where is the backing store location
|
||||
gpusize gpu_addr;
|
||||
void *cpu_addr;
|
||||
gpusize client_size; // user request size
|
||||
gpusize size;
|
||||
gpusize alignment;
|
||||
|
||||
union {
|
||||
struct {
|
||||
uint32_t is_virtual : 1;
|
||||
uint32_t is_shared : 1;
|
||||
uint32_t is_external : 1;
|
||||
uint32_t is_physical_only : 1;
|
||||
uint32_t is_locked : 1;
|
||||
|
||||
uint32_t unused : 27;
|
||||
};
|
||||
|
||||
uint32_t reserved;
|
||||
} flags;
|
||||
|
||||
int mem_flags;
|
||||
int engine_flag;
|
||||
};
|
||||
|
||||
struct SharedHandleInfo {
|
||||
rocr_proxy::AllocDomain domain;
|
||||
LUID adapter_luid;
|
||||
gpusize client_size; // user request size
|
||||
uint64_t size;
|
||||
uint32_t flags;
|
||||
int mem_flags;
|
||||
};
|
||||
|
||||
using GpuMemoryHandle = void *;
|
||||
|
||||
class GpuMemory {
|
||||
public:
|
||||
static size_t CalcChunkNumbers(gpusize size);
|
||||
|
||||
ErrorCode Init(const GpuMemoryCreateInfo &create_info);
|
||||
|
||||
WDDMDevice *GetDevice() const { return device_; }
|
||||
gpusize Size() const { return desc_.size; }
|
||||
gpusize ClientSize() const { return desc_.client_size; }
|
||||
uint64_t GpuAddress() const { return desc_.gpu_addr; }
|
||||
void *CpuAddress() const { return desc_.cpu_addr; }
|
||||
|
||||
inline bool IsLocal() const { return desc_.domain == rocr_proxy::kLocal; }
|
||||
inline bool IsUserMemory() const { return desc_.domain == rocr_proxy::kUserMemory; }
|
||||
inline bool IsSystem() const { return desc_.domain == rocr_proxy::kSystem; }
|
||||
inline bool IsUserQueue() const { return desc_.domain == rocr_proxy::kUserQueue; }
|
||||
inline bool IsPhysicalOnly() const { return desc_.flags.is_physical_only; }
|
||||
inline bool IsVirtual() const { return desc_.flags.is_virtual; }
|
||||
inline bool IsShared() const { return desc_.flags.is_shared; }
|
||||
inline bool IsExternal() const { return desc_.flags.is_external; }
|
||||
|
||||
inline uint32_t Flags() const { return desc_.flags.reserved; }
|
||||
inline int GetAllocInfo() const { return desc_.mem_flags; }
|
||||
inline bool IsFineGrain() const { return (desc_.mem_flags & rocr_proxy::kFineGrain); }
|
||||
inline bool IsSameAdapter(const LUID &luid) const {
|
||||
return (desc_.adapter_luid.HighPart == luid.HighPart &&
|
||||
desc_.adapter_luid.LowPart == luid.LowPart);
|
||||
}
|
||||
|
||||
WinAllocationHandle GetAllocationHandle(size_t index) const { return alloc_handles_ptr_[index]; }
|
||||
size_t NumChunks() const { return num_allocations_; }
|
||||
|
||||
const GpuMemoryHandle GetGpuMemoryHandle() const {
|
||||
return reinterpret_cast<GpuMemoryHandle>(const_cast<GpuMemory*>(this));
|
||||
}
|
||||
|
||||
static GpuMemory *Convert(GpuMemoryHandle handle) { return reinterpret_cast<GpuMemory *>(handle); }
|
||||
|
||||
ErrorCode ReserveGpuVirtualAddress(gpusize base_virt_addr, gpusize va_size, gpusize alignment);
|
||||
ErrorCode FreeGpuVirtualAddress(gpusize va_start_address, gpusize va_size);
|
||||
|
||||
ErrorCode MapGpuVirtualAddress(const gpusize map_addr, const gpusize size, gpusize offset = 0);
|
||||
ErrorCode UnmapGpuVirtualAddress(const gpusize map_addr, const gpusize size, gpusize offset = 0);
|
||||
|
||||
ErrorCode MakeResident();
|
||||
ErrorCode Evict();
|
||||
|
||||
ErrorCode ExportPhysicalHandle(int* dmabuf_fd, uint32_t flags = SHARED_ALLOCATION_ALL_ACCESS);
|
||||
ErrorCode ImportPhysicalHandle(int dmabuf_fd);
|
||||
~GpuMemory();
|
||||
protected:
|
||||
explicit GpuMemory(WDDMDevice *device);
|
||||
private:
|
||||
ErrorCode CreatePhysicalMemory();
|
||||
ErrorCode FreePhysicalMemory();
|
||||
|
||||
uint64_t AdjustSize(gpusize size) const;
|
||||
private:
|
||||
friend class WDDMDevice;
|
||||
|
||||
WDDMDevice *const device_;
|
||||
|
||||
GpuMemoryDesc desc_;
|
||||
|
||||
size_t num_allocations_;
|
||||
WinAllocationHandle *alloc_handles_ptr_;
|
||||
WinAllocationHandle alloc_handle_; // Optimization for num_allocations_ is 1
|
||||
|
||||
WinResourceHandle resource_; // Handle to a resource object that wraps the allocation. Used for shared resources
|
||||
|
||||
DISALLOW_COPY_AND_ASSIGN(GpuMemory);
|
||||
};
|
||||
|
||||
} // namespace core
|
||||
} // namespace rocr
|
||||
|
||||
#endif
|
||||
@@ -0,0 +1,284 @@
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
//
|
||||
// The University of Illinois/NCSA
|
||||
// Open Source License (NCSA)
|
||||
//
|
||||
// Copyright (c) 2020, Advanced Micro Devices, Inc. All rights reserved.
|
||||
//
|
||||
// Developed by:
|
||||
//
|
||||
// AMD Research and AMD HSA Software Development
|
||||
//
|
||||
// Advanced Micro Devices, Inc.
|
||||
//
|
||||
// www.amd.com
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
// of this software and associated documentation files (the "Software"), to
|
||||
// deal with the Software without restriction, including without limitation
|
||||
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
// and/or sell copies of the Software, and to permit persons to whom the
|
||||
// Software is furnished to do so, subject to the following conditions:
|
||||
//
|
||||
// - Redistributions of source code must retain the above copyright notice,
|
||||
// this list of conditions and the following disclaimers.
|
||||
// - Redistributions in binary form must reproduce the above copyright
|
||||
// notice, this list of conditions and the following disclaimers in
|
||||
// the documentation and/or other materials provided with the distribution.
|
||||
// - Neither the names of Advanced Micro Devices, Inc,
|
||||
// nor the names of its contributors may be used to endorse or promote
|
||||
// products derived from this Software without specific prior written
|
||||
// permission.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
|
||||
// THE CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
|
||||
// OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
|
||||
// ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
||||
// DEALINGS WITH THE SOFTWARE.
|
||||
//
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
#ifndef _WDDM_QUEUE_H_
|
||||
#define _WDDM_QUEUE_H_
|
||||
|
||||
#include <cinttypes>
|
||||
#include <condition_variable>
|
||||
#include "inc/wddm/types.h"
|
||||
#include "inc/wddm/device.h"
|
||||
#include "inc/wddm/gpu_memory.h"
|
||||
#include "hsa-runtime/inc/hsa_ext_amd.h"
|
||||
#include "hsa-runtime/inc/amd_hsa_queue.h"
|
||||
#include "hsa-runtime/inc/amd_hsa_signal.h"
|
||||
#include "inc/wddm/cmd_util.h"
|
||||
|
||||
namespace rocr {
|
||||
namespace core {
|
||||
|
||||
class Queue;
|
||||
class WDDMDevice;
|
||||
|
||||
class WDDMQueue {
|
||||
public:
|
||||
WDDMQueue(WDDMDevice *device,
|
||||
uint32_t cmdbuf_size,
|
||||
uint32_t engine,
|
||||
bool use_hws = true) :
|
||||
device(device),
|
||||
context(NULL),
|
||||
queue(NULL),
|
||||
syncobj(NULL),
|
||||
sync_addr(NULL),
|
||||
cmdbuf(0),
|
||||
cmdbuf_addr(0),
|
||||
cmdbuf_size(cmdbuf_size),
|
||||
queue_engine(engine),
|
||||
use_hws(use_hws),
|
||||
prio(rocr_proxy::kNormal) {
|
||||
|
||||
}
|
||||
|
||||
virtual ~WDDMQueue() { }
|
||||
|
||||
virtual hsa_status_t Init(void) = 0;
|
||||
virtual hsa_status_t Fini(void) = 0;
|
||||
|
||||
hsa_status_t SwsInit(void);
|
||||
hsa_status_t SwsFini(void);
|
||||
hsa_status_t SwsSubmit(uint64_t command_addr,
|
||||
uint64_t command_size,
|
||||
uint64_t fence_value);
|
||||
|
||||
hsa_status_t HwsInit(void);
|
||||
hsa_status_t HwsFini(void);
|
||||
hsa_status_t HwsSubmit(uint64_t command_addr,
|
||||
uint64_t command_size,
|
||||
uint64_t fence_value);
|
||||
hsa_status_t SetPriority(hsa_amd_queue_priority_t priority);
|
||||
|
||||
uint64_t *GetSyncAddr(void) const { return sync_addr; }
|
||||
uint64_t GetCmdbufAddr(void) const { return cmdbuf_addr; }
|
||||
|
||||
rocr_proxy::SchedLevel ConvertSchedLevel(hsa_amd_queue_priority_t prio) const {
|
||||
switch (prio) {
|
||||
case HSA_AMD_QUEUE_PRIORITY_LOW:
|
||||
return rocr_proxy::kLow;
|
||||
case HSA_AMD_QUEUE_PRIORITY_HIGH:
|
||||
return rocr_proxy::kHigh;
|
||||
case HSA_AMD_QUEUE_PRIORITY_NORMAL:
|
||||
default:
|
||||
return rocr_proxy::kNormal;
|
||||
}
|
||||
}
|
||||
|
||||
WDDMDevice *device;
|
||||
|
||||
D3DKMT_HANDLE context;
|
||||
D3DKMT_HANDLE queue;
|
||||
|
||||
D3DKMT_HANDLE syncobj;
|
||||
uint64_t *sync_addr;
|
||||
|
||||
GpuMemoryHandle cmdbuf;
|
||||
uint64_t cmdbuf_addr;
|
||||
uint32_t cmdbuf_size;
|
||||
|
||||
GpuMemoryHandle queue_mem;
|
||||
uint64_t queue_addr;
|
||||
|
||||
uint32_t queue_engine;
|
||||
|
||||
bool use_hws;
|
||||
rocr_proxy::SchedLevel prio;
|
||||
};
|
||||
|
||||
class ComputeQueue : public WDDMQueue {
|
||||
public:
|
||||
ComputeQueue(WDDMDevice *device,
|
||||
void *ring,
|
||||
uint64_t ring_size,
|
||||
std::atomic<uint64_t> *ring_wptr,
|
||||
std::atomic<uint64_t> *ring_rptr,
|
||||
volatile int64_t *error_addr,
|
||||
uint32_t cmdbuf_size,
|
||||
uint32_t engine,
|
||||
bool use_hws = true);
|
||||
|
||||
~ComputeQueue();
|
||||
|
||||
virtual hsa_status_t Init(void);
|
||||
virtual hsa_status_t Fini(void);
|
||||
virtual hsa_status_t Submit(void);
|
||||
|
||||
void* GetRing(void) const { return ring; }
|
||||
uint64_t GetRingSize(void) const { return ring_size; }
|
||||
std::atomic<uint64_t>* GetRingWptr(void) const { return ring_wptr; }
|
||||
std::atomic<uint64_t>* GetRingRptr(void) const { return ring_rptr; }
|
||||
|
||||
uint64_t GetAqlWriteIndex(void) const { return cmdbuf_aql_frame_write_index; }
|
||||
uint32_t GetAqlFrameSize(void) const { return cmdbuf_aql_frame_size; }
|
||||
|
||||
bool IsInvalidPacket(void) const {
|
||||
uint16_t *packet = (uint16_t *)((char *)ring +
|
||||
(cmdbuf_aql_frame_write_index % ring_size) * 64);
|
||||
return ((*packet >> HSA_PACKET_HEADER_TYPE) & ((1 << HSA_PACKET_HEADER_WIDTH_TYPE) - 1))
|
||||
== HSA_PACKET_TYPE_INVALID;
|
||||
}
|
||||
|
||||
hsa_status_t Process(void);
|
||||
uint64_t * GetDoorbellPtr() const { return (uint64_t *)&doorbell_signal_.value; }
|
||||
void RingDoorbell();
|
||||
private:
|
||||
hsa_status_t KernelDispatchAqlToPm4(char *cpu, hsa_kernel_dispatch_packet_t *packet);
|
||||
hsa_status_t BarrierGenericAqlToPm4(char *cpu, hsa_barrier_and_packet_t *packet, bool is_or = false);
|
||||
struct amd_aql_pm4_ib {
|
||||
uint16_t header;
|
||||
uint16_t ven_hdr;
|
||||
uint32_t ib_jump_cmd[4];
|
||||
uint32_t dw_cnt_remain;
|
||||
uint32_t reserved[8];
|
||||
hsa_signal_t completion_signal;
|
||||
};
|
||||
hsa_status_t VendorSpecificAqlToPm4(char *cpu, amd_aql_pm4_ib *packet);
|
||||
hsa_status_t SwitchAql2PM4(void);
|
||||
|
||||
hsa_status_t PreSubmit(void);
|
||||
hsa_status_t EndSubmit(void);
|
||||
|
||||
void *ring;
|
||||
uint64_t ring_size;
|
||||
std::atomic<uint64_t> *ring_wptr;
|
||||
std::atomic<uint64_t> *ring_rptr;
|
||||
|
||||
// ib_start_addr is the current ib start address
|
||||
uint64_t ib_start_addr;
|
||||
|
||||
// ib_size is the current ib size.
|
||||
uint64_t ib_size;
|
||||
|
||||
// record the last submitted aql frame write index
|
||||
uint64_t sync_point;
|
||||
|
||||
uint64_t cmdbuf_aql_frame_write_index;
|
||||
uint32_t cmdbuf_aql_frame_size;
|
||||
|
||||
bool needs_barrier;
|
||||
bool ready_to_submit;
|
||||
|
||||
CmdUtil cmd_util;
|
||||
|
||||
private:
|
||||
bool EnableProfiling() {
|
||||
return AMD_HSA_BITS_GET(amd_queue_rocr_->queue_properties, AMD_QUEUE_PROPERTIES_ENABLE_PROFILING);
|
||||
}
|
||||
void HandleError(hsa_status_t status);
|
||||
bool UpdateScratch(uint32_t private_segment_size, bool wave32);
|
||||
|
||||
uint32_t UpdateIndexStride(uint32_t srd, bool wave32);
|
||||
|
||||
void *ScratchBase() { return scratch_base_; }
|
||||
|
||||
void AppendCmdbufSratchBaseOffset(int offset) {
|
||||
scratch_base_offset_array_.push_back(offset);
|
||||
}
|
||||
|
||||
bool RelocateCmdbufScratchBase(uint64_t addr);
|
||||
|
||||
uint32_t ScratchSizePerWave() { return scratch_size_per_wave_; }
|
||||
uint64_t GetKernelObjAddr(uint64_t addr) const;
|
||||
void InitScratchSRD();
|
||||
GpuMemoryHandle amd_queue_mem_;
|
||||
amd_queue_t *amd_queue_;
|
||||
amd_queue_t *amd_queue_rocr_;
|
||||
amd_signal_t doorbell_signal_;
|
||||
volatile std::atomic<int64_t> *error_code_;
|
||||
std::thread aql_to_pm4_thread_;
|
||||
bool thread_stop_;
|
||||
std::mutex thread_cond_lock_;
|
||||
std::condition_variable thread_cond_;
|
||||
static void AqlToPm4Thread(ComputeQueue *queue);
|
||||
|
||||
uint32_t scratch_waves_;
|
||||
uint32_t scratch_size_per_wave_;
|
||||
uint32_t scratch_size_;
|
||||
void *scratch_base_;
|
||||
GpuMemoryHandle scratch_mem_;
|
||||
|
||||
std::vector<int> scratch_base_offset_array_;
|
||||
};
|
||||
|
||||
class SDMAQueue : public WDDMQueue {
|
||||
public:
|
||||
SDMAQueue(WDDMDevice *device,
|
||||
uint64_t cmdbuf_size,
|
||||
uint32_t engine,
|
||||
bool use_hws = true) :
|
||||
WDDMQueue(device, cmdbuf_size, engine, use_hws),
|
||||
rptr_next(0),
|
||||
ib_size(0),
|
||||
ib_start_addr(0) {
|
||||
|
||||
}
|
||||
|
||||
virtual ~SDMAQueue() { }
|
||||
|
||||
hsa_status_t Init(void);
|
||||
hsa_status_t Fini(void);
|
||||
hsa_status_t Submit(void);
|
||||
|
||||
int PreparePacket(uint32_t offset, uint64_t size);
|
||||
|
||||
void WaitQueue(void) {
|
||||
device->CpuWait(&syncobj, &rptr_next, 1, false);
|
||||
}
|
||||
|
||||
private:
|
||||
uint64_t rptr_next;
|
||||
uint64_t ib_size;
|
||||
uint64_t ib_start_addr;
|
||||
};
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
#endif
|
||||
@@ -0,0 +1,59 @@
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
//
|
||||
// The University of Illinois/NCSA
|
||||
// Open Source License (NCSA)
|
||||
//
|
||||
// Copyright (c) 2020, Advanced Micro Devices, Inc. All rights reserved.
|
||||
//
|
||||
// Developed by:
|
||||
//
|
||||
// AMD Research and AMD HSA Software Development
|
||||
//
|
||||
// Advanced Micro Devices, Inc.
|
||||
//
|
||||
// www.amd.com
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
// of this software and associated documentation files (the "Software"), to
|
||||
// deal with the Software without restriction, including without limitation
|
||||
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
// and/or sell copies of the Software, and to permit persons to whom the
|
||||
// Software is furnished to do so, subject to the following conditions:
|
||||
//
|
||||
// - Redistributions of source code must retain the above copyright notice,
|
||||
// this list of conditions and the following disclaimers.
|
||||
// - Redistributions in binary form must reproduce the above copyright
|
||||
// notice, this list of conditions and the following disclaimers in
|
||||
// the documentation and/or other materials provided with the distribution.
|
||||
// - Neither the names of Advanced Micro Devices, Inc,
|
||||
// nor the names of its contributors may be used to endorse or promote
|
||||
// products derived from this Software without specific prior written
|
||||
// permission.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
|
||||
// THE CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
|
||||
// OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
|
||||
// ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
||||
// DEALINGS WITH THE SOFTWARE.
|
||||
//
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
#ifndef _ROCR_CORE_INC_WDDM_STATUS_H_
|
||||
#define _ROCR_CORE_INC_WDDM_STATUS_H_
|
||||
|
||||
enum class ErrorCode {
|
||||
Success,
|
||||
DeviceLost,
|
||||
UnSupported,
|
||||
NotReady,
|
||||
OutOfMemory,
|
||||
OutOfGpuMemory,
|
||||
Timeout,
|
||||
SyscallFail,
|
||||
InvalidateParams,
|
||||
Unknown,
|
||||
};
|
||||
|
||||
#endif
|
||||
@@ -0,0 +1,232 @@
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
//
|
||||
// The University of Illinois/NCSA
|
||||
// Open Source License (NCSA)
|
||||
//
|
||||
// Copyright (c) 2020, Advanced Micro Devices, Inc. All rights reserved.
|
||||
//
|
||||
// Developed by:
|
||||
//
|
||||
// AMD Research and AMD HSA Software Development
|
||||
//
|
||||
// Advanced Micro Devices, Inc.
|
||||
//
|
||||
// www.amd.com
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
// of this software and associated documentation files (the "Software"), to
|
||||
// deal with the Software without restriction, including without limitation
|
||||
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
// and/or sell copies of the Software, and to permit persons to whom the
|
||||
// Software is furnished to do so, subject to the following conditions:
|
||||
//
|
||||
// - Redistributions of source code must retain the above copyright notice,
|
||||
// this list of conditions and the following disclaimers.
|
||||
// - Redistributions in binary form must reproduce the above copyright
|
||||
// notice, this list of conditions and the following disclaimers in
|
||||
// the documentation and/or other materials provided with the distribution.
|
||||
// - Neither the names of Advanced Micro Devices, Inc,
|
||||
// nor the names of its contributors may be used to endorse or promote
|
||||
// products derived from this Software without specific prior written
|
||||
// permission.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
|
||||
// THE CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
|
||||
// OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
|
||||
// ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
||||
// DEALINGS WITH THE SOFTWARE.
|
||||
//
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
#ifndef _ROCR_CORE_INC_WDDM_THUNKS_H_
|
||||
#define _ROCR_CORE_INC_WDDM_THUNKS_H_
|
||||
|
||||
#include "inc/wddm/status.h"
|
||||
#include "inc/wddm/types.h"
|
||||
|
||||
namespace rocr {
|
||||
namespace core {
|
||||
|
||||
inline ErrorCode TranslateNtStatus(NTSTATUS status) {
|
||||
switch (status) {
|
||||
case STATUS_SUCCESS:
|
||||
return ErrorCode::Success;
|
||||
case STATUS_PENDING:
|
||||
return ErrorCode::NotReady;
|
||||
case STATUS_NO_MEMORY:
|
||||
return ErrorCode::OutOfMemory;
|
||||
case STATUS_DEVICE_REMOVED:
|
||||
return ErrorCode::DeviceLost;
|
||||
case STATUS_GRAPHICS_NO_VIDEO_MEMORY:
|
||||
return ErrorCode::OutOfGpuMemory;
|
||||
case STATUS_TIMEOUT:
|
||||
return ErrorCode::Timeout;
|
||||
case STATUS_INVALID_PARAMETER:
|
||||
return ErrorCode::InvalidateParams;
|
||||
default:
|
||||
break;
|
||||
}
|
||||
return ErrorCode::Unknown;
|
||||
}
|
||||
|
||||
namespace thunk {
|
||||
|
||||
typedef D3DKMT_CREATEALLOCATION CreateAllocationArgs;
|
||||
typedef D3DKMT_CREATECONTEXT CreateContextArgs;
|
||||
typedef D3DKMT_CREATECONTEXTVIRTUAL CreateContextVirtualArgs;
|
||||
typedef D3DKMT_CREATEPAGINGQUEUE CreatePagingQueueArgs;
|
||||
typedef D3DKMT_CREATESYNCHRONIZATIONOBJECT CreateSynchronizationObjectArgs;
|
||||
typedef D3DKMT_CREATESYNCHRONIZATIONOBJECT2 CreateSynchronizationObject2Args;
|
||||
typedef D3DKMT_ESCAPE EscapeArgs;
|
||||
typedef D3DKMT_EVICT EvictArgs;
|
||||
typedef D3DKMT_FREEGPUVIRTUALADDRESS FreeGpuVirtualAddressArgs;
|
||||
typedef D3DKMT_LOCK LockArgs;
|
||||
typedef D3DKMT_LOCK2 Lock2Args;
|
||||
typedef D3DKMT_OPENRESOURCE OpenResourceArgs;
|
||||
typedef D3DKMT_OPENRESOURCEFROMNTHANDLE OpenResourceFromNtHandleArgs;
|
||||
typedef D3DKMT_QUERYADAPTERINFO QueryAdapterInfoArgs;
|
||||
typedef D3DKMT_SIGNALSYNCHRONIZATIONOBJECT SignalSynchronizationObjectArgs;
|
||||
typedef D3DKMT_SIGNALSYNCHRONIZATIONOBJECT2 SignalSynchronizationObject2Args;
|
||||
typedef D3DKMT_SIGNALSYNCHRONIZATIONOBJECTFROMCPU SignalSynchronizationObjectFromCpuArgs;
|
||||
typedef D3DKMT_SIGNALSYNCHRONIZATIONOBJECTFROMGPU2 SignalSynchronizationObjectFromGpuArgs;
|
||||
typedef D3DKMT_SUBMITCOMMAND SubmitCommandArgs;
|
||||
typedef D3DKMT_UNLOCK UnlockArgs;
|
||||
typedef D3DKMT_UNLOCK2 Unlock2Args;
|
||||
typedef D3DKMT_UPDATEGPUVIRTUALADDRESS UpdateGpuVirtualAddressArgs;
|
||||
typedef D3DKMT_WAITFORSYNCHRONIZATIONOBJECT WaitForSynchronizationObjectArgs;
|
||||
typedef D3DKMT_WAITFORSYNCHRONIZATIONOBJECT2 WaitForSynchronizationObject2Args;
|
||||
typedef D3DKMT_WAITFORSYNCHRONIZATIONOBJECTFROMCPU WaitForSynchronizationObjectFromCpuArgs;
|
||||
typedef D3DKMT_WAITFORSYNCHRONIZATIONOBJECTFROMGPU WaitForSynchronizationObjectFromGpuArgs;
|
||||
typedef D3DKMT_ACQUIREKEYEDMUTEX AcquireKeyedMutexArgs;
|
||||
typedef D3DKMT_RELEASEKEYEDMUTEX ReleaseKeyedMutexArgs;
|
||||
typedef D3DKMT_OPENKEYEDMUTEX OpenKeyedMutexArgs;
|
||||
typedef D3DKMT_DESTROYKEYEDMUTEX DestroyKeyedMutexArgs;
|
||||
typedef D3DKMT_QUERYVIDEOMEMORYINFO QueryVideoMemoryInfoArgs;
|
||||
typedef D3DKMT_CREATEHWQUEUE CreateHwQueueArgs;
|
||||
typedef D3DKMT_DESTROYHWQUEUE DestroyHwQueueArgs;
|
||||
typedef D3DKMT_SUBMITCOMMANDTOHWQUEUE SubmitCommandToHwQueueArgs;
|
||||
typedef D3DKMT_SUBMITPRESENTTOHWQUEUE SubmitPresentToHwQueueArgs;
|
||||
typedef D3DKMT_SUBMITSIGNALSYNCOBJECTSTOHWQUEUE SubmitSignalSyncObjectsToHwQueueArgs;
|
||||
typedef D3DKMT_SUBMITWAITFORSYNCOBJECTSTOHWQUEUE SubmitWaitForSyncObjectsToHwQueueArgs;
|
||||
typedef D3DKMT_CREATESYNCFILE CreateSyncFileArgs;
|
||||
|
||||
inline ErrorCode MapGpuVirtualAddress(D3DDDI_MAPGPUVIRTUALADDRESS *args) {
|
||||
return TranslateNtStatus(D3DKMTMapGpuVirtualAddress(args));
|
||||
}
|
||||
|
||||
inline ErrorCode CreateAllocation(CreateAllocationArgs *args) {
|
||||
return TranslateNtStatus(D3DKMTCreateAllocation2(args));
|
||||
}
|
||||
|
||||
inline ErrorCode DestroyAllocation(
|
||||
WinDeviceHandle device,
|
||||
WinResourceHandle resource,
|
||||
size_t num_allocations,
|
||||
const WinAllocationHandle *alloc_handles) {
|
||||
|
||||
D3DKMT_DESTROYALLOCATION2 args{};
|
||||
|
||||
memset(&args, 0, sizeof(args));
|
||||
args.hDevice = device;
|
||||
if (resource) {
|
||||
args.hResource = resource;
|
||||
} else {
|
||||
args.phAllocationList = alloc_handles;
|
||||
args.AllocationCount = num_allocations;
|
||||
}
|
||||
|
||||
return TranslateNtStatus(D3DKMTDestroyAllocation2(&args));
|
||||
}
|
||||
|
||||
inline ErrorCode ReserveGpuVirtualAddress(D3DDDI_RESERVEGPUVIRTUALADDRESS *args) {
|
||||
return TranslateNtStatus(D3DKMTReserveGpuVirtualAddress(args));
|
||||
}
|
||||
|
||||
inline ErrorCode ReserveGpuVirtualAddress(WinAdapterHandle handle,
|
||||
gpusize size,
|
||||
gpusize base_address,
|
||||
gpusize *out_addr) {
|
||||
D3DDDI_RESERVEGPUVIRTUALADDRESS args{};
|
||||
args.hPagingQueue = handle;
|
||||
args.Size = size;
|
||||
args.BaseAddress = base_address;
|
||||
|
||||
auto code = ReserveGpuVirtualAddress(&args);
|
||||
if (code == ErrorCode::Success)
|
||||
*out_addr = args.VirtualAddress;
|
||||
return code;
|
||||
}
|
||||
|
||||
inline ErrorCode ReserveGpuVirtualAddress(WinAdapterHandle handle,
|
||||
gpusize size,
|
||||
gpusize minimum_address,
|
||||
gpusize maximum_address,
|
||||
gpusize *out_addr) {
|
||||
D3DDDI_RESERVEGPUVIRTUALADDRESS args{};
|
||||
args.hPagingQueue = handle;
|
||||
args.Size = size;
|
||||
args.MinimumAddress = minimum_address;
|
||||
args.MaximumAddress = maximum_address;
|
||||
|
||||
auto code = ReserveGpuVirtualAddress(&args);
|
||||
if (code == ErrorCode::Success)
|
||||
*out_addr = args.VirtualAddress;
|
||||
return code;
|
||||
}
|
||||
|
||||
inline ErrorCode FreeGpuVirtualAddress(FreeGpuVirtualAddressArgs *args) {
|
||||
return TranslateNtStatus(D3DKMTFreeGpuVirtualAddress(args));
|
||||
}
|
||||
|
||||
inline ErrorCode FreeGpuVirtualAddress(WinAdapterHandle handle,
|
||||
gpusize base_address,
|
||||
gpusize size) {
|
||||
FreeGpuVirtualAddressArgs args{};
|
||||
args.hAdapter = handle;
|
||||
args.Size = size;
|
||||
args.BaseAddress = base_address;
|
||||
return FreeGpuVirtualAddress(&args);
|
||||
}
|
||||
|
||||
inline ErrorCode MakeResident(D3DDDI_MAKERESIDENT *args) {
|
||||
return TranslateNtStatus(D3DKMTMakeResident(args));
|
||||
}
|
||||
|
||||
inline ErrorCode Evict(EvictArgs *args) {
|
||||
return TranslateNtStatus(D3DKMTEvict(args));
|
||||
}
|
||||
|
||||
inline ErrorCode ShareObjects(size_t num_allocations,
|
||||
WinResourceHandle resource,
|
||||
uint32_t flags,
|
||||
int* dmabuf_fd) {
|
||||
OBJECT_ATTRIBUTES obj_attr;
|
||||
HANDLE nt_handle;
|
||||
ErrorCode ret;
|
||||
|
||||
InitializeObjectAttributes(&obj_attr, nullptr, OBJ_INHERIT, nullptr, nullptr);
|
||||
ret = TranslateNtStatus(D3DKMTShareObjects(num_allocations,
|
||||
&resource, &obj_attr, flags, &nt_handle));
|
||||
if (ret == ErrorCode::Success)
|
||||
*dmabuf_fd = *(reinterpret_cast<int*>(&nt_handle));
|
||||
else
|
||||
*dmabuf_fd = -1;
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
inline ErrorCode QueryResourceInfoFromNtHandle(D3DKMT_QUERYRESOURCEINFOFROMNTHANDLE *args) {
|
||||
return TranslateNtStatus(D3DKMTQueryResourceInfoFromNtHandle(args));
|
||||
}
|
||||
|
||||
inline ErrorCode OpenResourceFromNtHandle(D3DKMT_OPENRESOURCEFROMNTHANDLE *args) {
|
||||
return TranslateNtStatus(D3DKMTOpenResourceFromNtHandle(args));
|
||||
}
|
||||
|
||||
} // namespace thunk
|
||||
} // namespace core
|
||||
} // namespace rocr
|
||||
|
||||
#endif // _ROCR_CORE_INC_WDDM_THUNKS_H_
|
||||
@@ -0,0 +1,101 @@
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
//
|
||||
// The University of Illinois/NCSA
|
||||
// Open Source License (NCSA)
|
||||
//
|
||||
// Copyright (c) 2020, Advanced Micro Devices, Inc. All rights reserved.
|
||||
//
|
||||
// Developed by:
|
||||
//
|
||||
// AMD Research and AMD HSA Software Development
|
||||
//
|
||||
// Advanced Micro Devices, Inc.
|
||||
//
|
||||
// www.amd.com
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
// of this software and associated documentation files (the "Software"), to
|
||||
// deal with the Software without restriction, including without limitation
|
||||
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
// and/or sell copies of the Software, and to permit persons to whom the
|
||||
// Software is furnished to do so, subject to the following conditions:
|
||||
//
|
||||
// - Redistributions of source code must retain the above copyright notice,
|
||||
// this list of conditions and the following disclaimers.
|
||||
// - Redistributions in binary form must reproduce the above copyright
|
||||
// notice, this list of conditions and the following disclaimers in
|
||||
// the documentation and/or other materials provided with the distribution.
|
||||
// - Neither the names of Advanced Micro Devices, Inc,
|
||||
// nor the names of its contributors may be used to endorse or promote
|
||||
// products derived from this Software without specific prior written
|
||||
// permission.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
|
||||
// THE CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
|
||||
// OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
|
||||
// ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
||||
// DEALINGS WITH THE SOFTWARE.
|
||||
//
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
#ifndef _ROCR_CORE_INC_WDDM_TYPES_H_
|
||||
#define _ROCR_CORE_INC_WDDM_TYPES_H_
|
||||
|
||||
#include <cstdint>
|
||||
#include <ntstatus.h>
|
||||
#include "inc/rocr_proxy/wddm_types.h"
|
||||
// windows wchar is 16bit, but linux is 32bit
|
||||
// seems libdxcore (not dxgkrnl.ko) convert thunk windows wchar to linux one
|
||||
// so only accept 32bit wchar args. note driver private data structure still
|
||||
// use 16bit wchar
|
||||
#define WCHAR wchar_t
|
||||
#define PCWSTR const wchar_t *
|
||||
#include <d3dkmthk.h>
|
||||
#undef WCHAR
|
||||
#undef PCWSTR
|
||||
|
||||
using gpusize = uint64_t; // Used to specify GPU addresses and sizes of GPU allocations
|
||||
using WinAllocationHandle = D3DKMT_HANDLE;
|
||||
using WinResourceHandle = D3DKMT_HANDLE;
|
||||
using WinContextHandle = D3DKMT_HANDLE;
|
||||
using WinDeviceHandle = D3DKMT_HANDLE;
|
||||
using WinAdapterHandle = D3DKMT_HANDLE;
|
||||
|
||||
//reference dk/winnt.h
|
||||
#define STANDARD_RIGHTS_REQUIRED (0x000F0000L)
|
||||
|
||||
//reference dk/ntdef.h
|
||||
#define OBJ_INHERIT (0x00000002L)
|
||||
typedef WCHAR *PWCHAR, *LPWCH, *PWCH;
|
||||
typedef struct _UNICODE_STRING {
|
||||
USHORT Length;
|
||||
USHORT MaximumLength;
|
||||
#ifdef MIDL_PASS
|
||||
[size_is(MaximumLength / 2), length_is((Length) / 2) ] USHORT * Buffer;
|
||||
#else // MIDL_PASS
|
||||
_Field_size_bytes_part_opt_(MaximumLength, Length) PWCH Buffer;
|
||||
#endif // MIDL_PASS
|
||||
} UNICODE_STRING;
|
||||
typedef UNICODE_STRING *PUNICODE_STRING;
|
||||
typedef const UNICODE_STRING *PCUNICODE_STRING;
|
||||
|
||||
typedef struct _OBJECT_ATTRIBUTES {
|
||||
ULONG Length;
|
||||
HANDLE RootDirectory;
|
||||
PUNICODE_STRING ObjectName;
|
||||
ULONG Attributes;
|
||||
PVOID SecurityDescriptor;
|
||||
PVOID SecurityQualityOfService;
|
||||
} OBJECT_ATTRIBUTES;
|
||||
#define InitializeObjectAttributes( p, n, a, r, s ) { \
|
||||
(p)->Length = sizeof( OBJECT_ATTRIBUTES ); \
|
||||
(p)->RootDirectory = r; \
|
||||
(p)->Attributes = a; \
|
||||
(p)->ObjectName = n; \
|
||||
(p)->SecurityDescriptor = s; \
|
||||
(p)->SecurityQualityOfService = NULL; \
|
||||
}
|
||||
|
||||
#endif // _ROCR_CORE_INC_WDDM_TYPES_H_
|
||||
@@ -0,0 +1,86 @@
|
||||
#ifndef VA_MGR_H_
|
||||
#define VA_MGR_H_
|
||||
|
||||
#include <mutex>
|
||||
#include <map>
|
||||
#include "util/utils.h"
|
||||
|
||||
namespace rocr {
|
||||
namespace core {
|
||||
|
||||
class VaMgr {
|
||||
public:
|
||||
VaMgr(uint64_t start, uint64_t size, uint64_t min_align);
|
||||
~VaMgr();
|
||||
|
||||
/* Allocate `bytes` VA, if `align` is not zero, the returned address is aligned by `align`.
|
||||
* If `addr` parameter is not zero, try best to allocate VA from fixed address `addr`.
|
||||
*/
|
||||
uint64_t Alloc(uint64_t bytes, uint64_t align, uint64_t addr = 0);
|
||||
|
||||
void Free(uint64_t addr);
|
||||
|
||||
private:
|
||||
uint64_t AllocImpl(uint64_t bytes, uint64_t align);
|
||||
|
||||
struct Fragment {
|
||||
using ptr = std::multimap<uint64_t, uint64_t>::iterator;
|
||||
ptr free_list_entry_;
|
||||
|
||||
struct {
|
||||
uint64_t size : 63;
|
||||
bool is_free : 1;
|
||||
};
|
||||
|
||||
Fragment() : size(0), is_free(false) {}
|
||||
Fragment(ptr iterator, uint64_t len, bool is_free)
|
||||
: free_list_entry_(iterator), size(len), is_free(is_free) {}
|
||||
};
|
||||
|
||||
static inline Fragment make_fragment(typename Fragment::ptr iter, uint64_t len) {
|
||||
return {iter, len, true};
|
||||
}
|
||||
|
||||
inline Fragment make_fragment(uint64_t len) { return {free_list_.end(), len, false}; }
|
||||
|
||||
static inline bool is_free(const Fragment& f) { return f.is_free; }
|
||||
void set_used(Fragment& f) {
|
||||
f.is_free = false;
|
||||
f.free_list_entry_ = free_list_.end();
|
||||
}
|
||||
static void set_free(Fragment& f, typename Fragment::ptr iter) {
|
||||
f.free_list_entry_ = iter;
|
||||
f.is_free = true;
|
||||
}
|
||||
|
||||
inline void remove_free_list_entry(Fragment& frag) {
|
||||
if (frag.free_list_entry_ != free_list_.end()) {
|
||||
free_list_.erase(frag.free_list_entry_);
|
||||
frag.free_list_entry_ = free_list_.end();
|
||||
}
|
||||
}
|
||||
|
||||
inline void add_free_fragment(uint64_t size, uint64_t base) {
|
||||
auto it = free_list_.insert(std::make_pair(size, base));
|
||||
frag_map_[base] = make_fragment(it, size);
|
||||
}
|
||||
|
||||
inline void add_used_fragment(uint64_t size, uint64_t base) {
|
||||
frag_map_[base] = make_fragment(size);
|
||||
}
|
||||
// Indexed by size
|
||||
std::multimap<uint64_t, uint64_t> free_list_;
|
||||
// Indexed by VA, each fragment has no overlap
|
||||
std::map<uint64_t, Fragment> frag_map_;
|
||||
|
||||
uint64_t min_align_;
|
||||
|
||||
std::mutex lock_; // Mutex protecting allocation and free of va
|
||||
|
||||
|
||||
DISALLOW_COPY_AND_ASSIGN(VaMgr);
|
||||
};
|
||||
|
||||
} // namespace core
|
||||
} // namespace rocr
|
||||
#endif // VA_MGR_H_
|
||||
@@ -0,0 +1,69 @@
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
//
|
||||
// The University of Illinois/NCSA
|
||||
// Open Source License (NCSA)
|
||||
//
|
||||
// Copyright (c) 2020, Advanced Micro Devices, Inc. All rights reserved.
|
||||
//
|
||||
// Developed by:
|
||||
//
|
||||
// AMD Research and AMD HSA Software Development
|
||||
//
|
||||
// Advanced Micro Devices, Inc.
|
||||
//
|
||||
// www.amd.com
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
// of this software and associated documentation files (the "Software"), to
|
||||
// deal with the Software without restriction, including without limitation
|
||||
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
// and/or sell copies of the Software, and to permit persons to whom the
|
||||
// Software is furnished to do so, subject to the following conditions:
|
||||
//
|
||||
// - Redistributions of source code must retain the above copyright notice,
|
||||
// this list of conditions and the following disclaimers.
|
||||
// - Redistributions in binary form must reproduce the above copyright
|
||||
// notice, this list of conditions and the following disclaimers in
|
||||
// the documentation and/or other materials provided with the distribution.
|
||||
// - Neither the names of Advanced Micro Devices, Inc,
|
||||
// nor the names of its contributors may be used to endorse or promote
|
||||
// products derived from this Software without specific prior written
|
||||
// permission.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
|
||||
// THE CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
|
||||
// OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
|
||||
// ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
||||
// DEALINGS WITH THE SOFTWARE.
|
||||
//
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
#include <cstdint>
|
||||
#include <amdgpu.h>
|
||||
#include <amdgpu_drm.h>
|
||||
|
||||
#include "inc/wddm/types.h"
|
||||
#include "inc/wddm/device.h"
|
||||
#include "libhsakmt.h"
|
||||
|
||||
HSAKMT_STATUS HSAKMTAPI hsaKmtGetAMDGPUDeviceHandle(
|
||||
HSAuint32 NodeId, HsaAMDGPUDeviceHandle *DeviceHandle) {
|
||||
CHECK_DXG_OPEN();
|
||||
|
||||
rocr::core::WDDMDevice *pDevice = get_wddmdev(NodeId);
|
||||
if (pDevice != nullptr) {
|
||||
*DeviceHandle = reinterpret_cast<HsaAMDGPUDeviceHandle>(pDevice);
|
||||
return HSAKMT_STATUS_SUCCESS;
|
||||
}
|
||||
return HSAKMT_STATUS_ERROR;
|
||||
}
|
||||
|
||||
HSAKMTAPI int hsaKmtamdgpu_query_gpu_info(void *dev,
|
||||
struct amdgpu_gpu_info *info) {
|
||||
rocr::core::WDDMDevice *pDevice =
|
||||
reinterpret_cast<rocr::core::WDDMDevice *>(dev);
|
||||
memset(info, 0, sizeof(*info));
|
||||
info->gpu_counter_freq = pDevice->GPUCounterFrequency() / 1000ull;
|
||||
return 0;
|
||||
}
|
||||
+158
@@ -0,0 +1,158 @@
|
||||
/*
|
||||
* Copyright © 2014 Advanced Micro Devices, Inc.
|
||||
*
|
||||
* Permission is hereby granted, free of charge, to any person
|
||||
* obtaining a copy of this software and associated documentation
|
||||
* files (the "Software"), to deal in the Software without
|
||||
* restriction, including without limitation the rights to use, copy,
|
||||
* modify, merge, publish, distribute, sublicense, and/or sell copies
|
||||
* of the Software, and to permit persons to whom the Software is
|
||||
* furnished to do so, subject to the following conditions:
|
||||
*
|
||||
* The above copyright notice and this permission notice (including
|
||||
* the next paragraph) shall be included in all copies or substantial
|
||||
* portions of the Software.
|
||||
*
|
||||
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
||||
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
||||
* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
||||
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
|
||||
* HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
|
||||
* WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
||||
* DEALINGS IN THE SOFTWARE.
|
||||
*/
|
||||
|
||||
#ifndef LIBHSAKMT_H_INCLUDED
|
||||
#define LIBHSAKMT_H_INCLUDED
|
||||
|
||||
#include <pthread.h>
|
||||
#include <stdint.h>
|
||||
#include <limits.h>
|
||||
#include "hsakmt/hsakmt.h"
|
||||
|
||||
#include "inc/wddm/types.h"
|
||||
#include "inc/wddm/device.h"
|
||||
|
||||
rocr::core::WDDMDevice* get_wddmdev(uint32_t node_id);
|
||||
|
||||
extern unsigned long dxg_open_count;
|
||||
extern bool hsakmt_forked;
|
||||
extern pthread_mutex_t hsakmt_mutex;
|
||||
extern bool is_dgpu;
|
||||
extern bool is_svm_api_supported;
|
||||
extern int zfb_support;
|
||||
extern int vendor_packet_support;
|
||||
|
||||
#undef HSAKMTAPI
|
||||
#define HSAKMTAPI __attribute__((visibility ("default")))
|
||||
|
||||
#if defined(__clang__)
|
||||
#if __has_feature(address_sanitizer)
|
||||
#define SANITIZER_AMDGPU 1
|
||||
#endif
|
||||
#endif
|
||||
|
||||
/*Avoid pointer-to-int-cast warning*/
|
||||
#define PORT_VPTR_TO_UINT64(vptr) ((uint64_t)(unsigned long)(vptr))
|
||||
|
||||
/*Avoid int-to-pointer-cast warning*/
|
||||
#define PORT_UINT64_TO_VPTR(v) ((void*)(unsigned long)(v))
|
||||
|
||||
#define CHECK_DXG_OPEN() \
|
||||
do { if (dxg_open_count == 0 || hsakmt_forked) return HSAKMT_STATUS_KERNEL_IO_CHANNEL_NOT_OPENED; } while (0)
|
||||
|
||||
/* Might be defined in limits.h on platforms where it is constant (used by musl) */
|
||||
/* See also: https://pubs.opengroup.org/onlinepubs/7908799/xsh/limits.h.html */
|
||||
#ifndef PAGE_SIZE
|
||||
extern int PAGE_SIZE;
|
||||
#endif
|
||||
extern int PAGE_SHIFT;
|
||||
|
||||
/* 64KB BigK fragment size for TLB efficiency */
|
||||
#define GPU_BIGK_PAGE_SIZE (1 << 16)
|
||||
|
||||
/* 2MB huge page size for 4-level page tables on Vega10 and later GPUs */
|
||||
#define GPU_HUGE_PAGE_SIZE (2 << 20)
|
||||
|
||||
#define CHECK_PAGE_MULTIPLE(x) \
|
||||
do { if ((uint64_t)PORT_VPTR_TO_UINT64(x) % PAGE_SIZE) return HSAKMT_STATUS_INVALID_PARAMETER; } while(0)
|
||||
|
||||
#define ALIGN_UP(x,align) (((uint64_t)(x) + (align) - 1) & ~(uint64_t)((align)-1))
|
||||
#define ALIGN_UP_32(x,align) (((uint32_t)(x) + (align) - 1) & ~(uint32_t)((align)-1))
|
||||
#define PAGE_ALIGN_UP(x) ALIGN_UP(x,PAGE_SIZE)
|
||||
#define BITMASK(n) ((n) ? (UINT64_MAX >> (sizeof(UINT64_MAX) * CHAR_BIT - (n))) : 0)
|
||||
#define ARRAY_LEN(array) (sizeof(array) / sizeof(array[0]))
|
||||
|
||||
/* HSA Thunk logging usage */
|
||||
extern int hsakmt_debug_level;
|
||||
#define hsakmt_print(level, fmt, ...) \
|
||||
do { if (level <= hsakmt_debug_level) fprintf(stderr, fmt, ##__VA_ARGS__); } while (0)
|
||||
#define HSAKMT_DEBUG_LEVEL_DEFAULT -1
|
||||
#define HSAKMT_DEBUG_LEVEL_ERR 3
|
||||
#define HSAKMT_DEBUG_LEVEL_WARNING 4
|
||||
#define HSAKMT_DEBUG_LEVEL_INFO 6
|
||||
#define HSAKMT_DEBUG_LEVEL_DEBUG 7
|
||||
#define pr_err(fmt, ...) \
|
||||
hsakmt_print(HSAKMT_DEBUG_LEVEL_ERR, fmt, ##__VA_ARGS__)
|
||||
#define pr_warn(fmt, ...) \
|
||||
hsakmt_print(HSAKMT_DEBUG_LEVEL_WARNING, fmt, ##__VA_ARGS__)
|
||||
#define pr_info(fmt, ...) \
|
||||
hsakmt_print(HSAKMT_DEBUG_LEVEL_INFO, fmt, ##__VA_ARGS__)
|
||||
#define pr_debug(fmt, ...) \
|
||||
hsakmt_print(HSAKMT_DEBUG_LEVEL_DEBUG, fmt, ##__VA_ARGS__)
|
||||
#define pr_err_once(fmt, ...) \
|
||||
({ \
|
||||
static bool __print_once; \
|
||||
if (!__print_once) { \
|
||||
__print_once = true; \
|
||||
pr_err(fmt, ##__VA_ARGS__); \
|
||||
} \
|
||||
})
|
||||
#define pr_warn_once(fmt, ...) \
|
||||
({ \
|
||||
static bool __print_once; \
|
||||
if (!__print_once) { \
|
||||
__print_once = true; \
|
||||
pr_warn(fmt, ##__VA_ARGS__); \
|
||||
} \
|
||||
})
|
||||
|
||||
/* Expects HSA_ENGINE_ID.ui32, returns gfxv (full) in hex */
|
||||
#define HSA_GET_GFX_VERSION_FULL(ui32) \
|
||||
(((ui32.Major) << 16) | ((ui32.Minor) << 8) | (ui32.Stepping))
|
||||
|
||||
HSAKMT_STATUS validate_nodeid(uint32_t nodeid, uint32_t *gpu_id);
|
||||
HSAKMT_STATUS gpuid_to_nodeid(uint32_t gpu_id, uint32_t* node_id);
|
||||
bool prefer_ats(HSAuint32 node_id);
|
||||
uint16_t get_device_id_by_node_id(HSAuint32 node_id);
|
||||
uint16_t get_device_id_by_gpu_id(HSAuint32 gpu_id);
|
||||
uint32_t get_direct_link_cpu(uint32_t gpu_node);
|
||||
|
||||
HSAKMT_STATUS topology_sysfs_get_system_props(HsaSystemProperties *props);
|
||||
HSAKMT_STATUS topology_get_node_props(HSAuint32 NodeId,
|
||||
HsaNodeProperties *NodeProperties);
|
||||
HSAKMT_STATUS topology_get_iolink_props(HSAuint32 NodeId,
|
||||
HSAuint32 NumIoLinks,
|
||||
HsaIoLinkProperties *IoLinkProperties);
|
||||
void topology_setup_is_dgpu_param(HsaNodeProperties *props);
|
||||
|
||||
HSAuint32 PageSizeFromFlags(unsigned int pageSizeFlags);
|
||||
|
||||
#define MIN(a, b) ({ \
|
||||
typeof(a) tmp1 = (a), tmp2 = (b); \
|
||||
tmp1 < tmp2 ? tmp1 : tmp2; })
|
||||
|
||||
#define MAX(a, b) ({ \
|
||||
typeof(a) tmp1 = (a), tmp2 = (b); \
|
||||
tmp1 > tmp2 ? tmp1 : tmp2; })
|
||||
|
||||
uint32_t get_num_sysfs_nodes(void);
|
||||
|
||||
bool is_forked_child(void);
|
||||
|
||||
/* Calculate VGPR and SGPR register file size per CU */
|
||||
uint32_t get_vgpr_size_per_cu(HSA_ENGINE_ID id);
|
||||
#define SGPR_SIZE_PER_CU 0x4000
|
||||
|
||||
#endif
|
||||
@@ -0,0 +1,97 @@
|
||||
HSAKMT_1
|
||||
{
|
||||
global:
|
||||
hsaKmtOpenKFD;
|
||||
hsaKmtCloseKFD;
|
||||
hsaKmtGetVersion;
|
||||
hsaKmtAcquireSystemProperties;
|
||||
hsaKmtReleaseSystemProperties;
|
||||
hsaKmtGetNodeProperties;
|
||||
hsaKmtGetNodeMemoryProperties;
|
||||
hsaKmtGetNodeCacheProperties;
|
||||
hsaKmtGetNodeIoLinkProperties;
|
||||
hsaKmtCreateEvent;
|
||||
hsaKmtDestroyEvent;
|
||||
hsaKmtSetEvent;
|
||||
hsaKmtResetEvent;
|
||||
hsaKmtQueryEventState;
|
||||
hsaKmtWaitOnEvent;
|
||||
hsaKmtWaitOnMultipleEvents;
|
||||
hsaKmtCreateQueue;
|
||||
hsaKmtUpdateQueue;
|
||||
hsaKmtDestroyQueue;
|
||||
hsaKmtSetQueueCUMask;
|
||||
hsaKmtSetMemoryPolicy;
|
||||
hsaKmtAllocMemory;
|
||||
hsaKmtAllocMemoryAlign;
|
||||
hsaKmtFreeMemory;
|
||||
hsaKmtAvailableMemory;
|
||||
hsaKmtRegisterMemory;
|
||||
hsaKmtRegisterMemoryToNodes;
|
||||
hsaKmtRegisterMemoryWithFlags;
|
||||
hsaKmtRegisterGraphicsHandleToNodes;
|
||||
hsaKmtShareMemory;
|
||||
hsaKmtRegisterSharedHandle;
|
||||
hsaKmtRegisterSharedHandleToNodes;
|
||||
hsaKmtProcessVMRead;
|
||||
hsaKmtProcessVMWrite;
|
||||
hsaKmtDeregisterMemory;
|
||||
hsaKmtMapMemoryToGPU;
|
||||
hsaKmtMapMemoryToGPUNodes;
|
||||
hsaKmtUnmapMemoryToGPU;
|
||||
hsaKmtDbgRegister;
|
||||
hsaKmtDbgUnregister;
|
||||
hsaKmtDbgWavefrontControl;
|
||||
hsaKmtDbgAddressWatch;
|
||||
hsaKmtDbgEnable;
|
||||
hsaKmtDbgDisable;
|
||||
hsaKmtDbgGetDeviceData;
|
||||
hsaKmtDbgGetQueueData;
|
||||
hsaKmtGetClockCounters;
|
||||
hsaKmtPmcGetCounterProperties;
|
||||
hsaKmtPmcRegisterTrace;
|
||||
hsaKmtPmcUnregisterTrace;
|
||||
hsaKmtPmcAcquireTraceAccess;
|
||||
hsaKmtPmcReleaseTraceAccess;
|
||||
hsaKmtPmcStartTrace;
|
||||
hsaKmtPmcQueryTrace;
|
||||
hsaKmtPmcStopTrace;
|
||||
hsaKmtMapGraphicHandle;
|
||||
hsaKmtUnmapGraphicHandle;
|
||||
hsaKmtSetTrapHandler;
|
||||
hsaKmtGetTileConfig;
|
||||
hsaKmtQueryPointerInfo;
|
||||
hsaKmtSetMemoryUserData;
|
||||
hsaKmtGetQueueInfo;
|
||||
hsaKmtAllocQueueGWS;
|
||||
hsaKmtRuntimeEnable;
|
||||
hsaKmtRuntimeDisable;
|
||||
hsaKmtCheckRuntimeDebugSupport;
|
||||
hsaKmtGetRuntimeCapabilities;
|
||||
hsaKmtDebugTrapIoctl;
|
||||
hsaKmtSPMAcquire;
|
||||
hsaKmtSPMRelease;
|
||||
hsaKmtSPMSetDestBuffer;
|
||||
hsaKmtSVMSetAttr;
|
||||
hsaKmtSVMGetAttr;
|
||||
hsaKmtSetXNACKMode;
|
||||
hsaKmtGetXNACKMode;
|
||||
hsaKmtOpenSMI;
|
||||
hsaKmtExportDMABufHandle;
|
||||
hsaKmtWaitOnEvent_Ext;
|
||||
hsaKmtWaitOnMultipleEvents_Ext;
|
||||
hsaKmtReplaceAsanHeaderPage;
|
||||
hsaKmtReturnAsanHeaderPage;
|
||||
hsaKmtGetAMDGPUDeviceHandle;
|
||||
hsaKmtPcSamplingQueryCapabilities;
|
||||
hsaKmtPcSamplingCreate;
|
||||
hsaKmtPcSamplingDestroy;
|
||||
hsaKmtPcSamplingStart;
|
||||
hsaKmtPcSamplingStop;
|
||||
hsaKmtPcSamplingSupport;
|
||||
hsaKmtGetVersionCapInfo;
|
||||
hsaKmtQueueRingDoorbell;
|
||||
hsaKmtamdgpu_query_gpu_info;
|
||||
local: *;
|
||||
};
|
||||
|
||||
+554
@@ -0,0 +1,554 @@
|
||||
/*
|
||||
* Copyright © 2014 Advanced Micro Devices, Inc.
|
||||
*
|
||||
* Permission is hereby granted, free of charge, to any person
|
||||
* obtaining a copy of this software and associated documentation
|
||||
* files (the "Software"), to deal in the Software without
|
||||
* restriction, including without limitation the rights to use, copy,
|
||||
* modify, merge, publish, distribute, sublicense, and/or sell copies
|
||||
* of the Software, and to permit persons to whom the Software is
|
||||
* furnished to do so, subject to the following conditions:
|
||||
*
|
||||
* The above copyright notice and this permission notice (including
|
||||
* the next paragraph) shall be included in all copies or substantial
|
||||
* portions of the Software.
|
||||
*
|
||||
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
||||
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
||||
* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
||||
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
|
||||
* HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
|
||||
* WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
||||
* DEALINGS IN THE SOFTWARE.
|
||||
*/
|
||||
|
||||
#include "libhsakmt.h"
|
||||
#include <stdlib.h>
|
||||
#include <stdio.h>
|
||||
#include <string.h>
|
||||
#include <assert.h>
|
||||
#include <sys/types.h>
|
||||
#include <sys/mman.h>
|
||||
#include <sys/sysinfo.h>
|
||||
#include <fcntl.h>
|
||||
#include "inc/wddm/gpu_memory.h"
|
||||
|
||||
struct Allocation {
|
||||
Allocation()
|
||||
: handle(0), cpu_addr(0), gpu_addr(0), size(0), userptr(false),
|
||||
user_data(nullptr), size_requested(0), node_id(0), mem_flags_value(0) {}
|
||||
Allocation(rocr::core::GpuMemoryHandle handle_arg, void *cpu_addr_arg,
|
||||
uint64_t gpu_addr_arg, size_t size_arg, bool userptr_arg = false,
|
||||
void *user_data_arg = nullptr, size_t user_size_arg = 0,
|
||||
HSAuint32 node_id_arg = 0, HSAuint32 mem_flags_value_arg = 0)
|
||||
: handle(handle_arg), cpu_addr(cpu_addr_arg), gpu_addr(gpu_addr_arg),
|
||||
size(size_arg), userptr(userptr_arg), user_data(user_data_arg),
|
||||
size_requested(user_size_arg), node_id(node_id_arg),
|
||||
mem_flags_value(mem_flags_value_arg) {}
|
||||
|
||||
rocr::core::GpuMemoryHandle handle;
|
||||
void *cpu_addr;
|
||||
uint64_t gpu_addr;
|
||||
bool userptr;
|
||||
size_t size; /* actual size = align_up(size_requested, granularity) */
|
||||
void *user_data;
|
||||
size_t size_requested; /* size requested by user */
|
||||
HSAuint32 node_id;
|
||||
HSAuint32 mem_flags_value;
|
||||
};
|
||||
|
||||
static std::map<const void *, Allocation> allocation_map_;
|
||||
static std::mutex allocation_map_lock_;
|
||||
|
||||
HSAKMT_STATUS HSAKMTAPI hsaKmtSetMemoryPolicy(HSAuint32 Node,
|
||||
HSAuint32 DefaultPolicy,
|
||||
HSAuint32 AlternatePolicy,
|
||||
void *MemoryAddressAlternate,
|
||||
HSAuint64 MemorySizeInBytes) {
|
||||
CHECK_DXG_OPEN();
|
||||
|
||||
assert(false);
|
||||
return HSAKMT_STATUS_SUCCESS;
|
||||
}
|
||||
|
||||
HSAuint32 PageSizeFromFlags(unsigned int pageSizeFlags) {
|
||||
switch (pageSizeFlags) {
|
||||
case HSA_PAGE_SIZE_4KB:
|
||||
return 4 * 1024;
|
||||
case HSA_PAGE_SIZE_64KB:
|
||||
return 64 * 1024;
|
||||
case HSA_PAGE_SIZE_2MB:
|
||||
return 2 * 1024 * 1024;
|
||||
case HSA_PAGE_SIZE_1GB:
|
||||
return 1024 * 1024 * 1024;
|
||||
default:
|
||||
assert(false);
|
||||
return 4 * 1024;
|
||||
}
|
||||
}
|
||||
|
||||
HSAKMT_STATUS HSAKMTAPI hsaKmtAllocMemory(HSAuint32 PreferredNode,
|
||||
HSAuint64 SizeInBytes,
|
||||
HsaMemFlags MemFlags,
|
||||
void **MemoryAddress) {
|
||||
return hsaKmtAllocMemoryAlign(PreferredNode, SizeInBytes, 0, MemFlags,
|
||||
MemoryAddress);
|
||||
}
|
||||
|
||||
#define POWER_OF_2(x) ((x && (!(x & (x - 1)))) ? 1 : 0)
|
||||
|
||||
bool isSystemMemoryAvailable(HSAuint64 SizeInBytes) {
|
||||
struct sysinfo info;
|
||||
if (sysinfo(&info) != 0)
|
||||
return false;
|
||||
return SizeInBytes <= info.freeram;
|
||||
}
|
||||
|
||||
bool isLocalMemoryAvailable(rocr::core::WDDMDevice *dev,
|
||||
HSAuint64 SizeInBytes) {
|
||||
return SizeInBytes <= dev->VramAvail();
|
||||
}
|
||||
|
||||
HSAKMT_STATUS HSAKMTAPI hsaKmtAllocMemoryAlign(HSAuint32 PreferredNode,
|
||||
HSAuint64 SizeInBytes,
|
||||
HSAuint64 Alignment,
|
||||
HsaMemFlags MemFlags,
|
||||
void **MemoryAddress) {
|
||||
CHECK_DXG_OPEN();
|
||||
|
||||
if (!MemoryAddress)
|
||||
return HSAKMT_STATUS_INVALID_PARAMETER;
|
||||
|
||||
if (MemFlags.ui32.FixedAddress) {
|
||||
if (*MemoryAddress == nullptr)
|
||||
return HSAKMT_STATUS_INVALID_PARAMETER;
|
||||
} else
|
||||
*MemoryAddress = nullptr;
|
||||
|
||||
rocr::core::WDDMDevice *dev = get_wddmdev(1);
|
||||
if (!dev)
|
||||
return HSAKMT_STATUS_ERROR;
|
||||
|
||||
rocr::core::GpuMemory *gpu_mem = nullptr;
|
||||
rocr::core::GpuMemoryCreateInfo create_info{};
|
||||
create_info.size = SizeInBytes;
|
||||
|
||||
if (!MemFlags.ui32.NonPaged || zfb_support || MemFlags.ui32.GTTAccess) {
|
||||
/* If allocate VRAM under ZFB mode */
|
||||
if (zfb_support && MemFlags.ui32.NonPaged == 1)
|
||||
MemFlags.ui32.CoarseGrain = 1;
|
||||
|
||||
create_info.domain = rocr_proxy::AllocDomain::kSystem;
|
||||
if (!isSystemMemoryAvailable(SizeInBytes))
|
||||
return HSAKMT_STATUS_NO_MEMORY;
|
||||
} else {
|
||||
create_info.domain = rocr_proxy::AllocDomain::kLocal;
|
||||
if (!isLocalMemoryAvailable(dev, SizeInBytes))
|
||||
return HSAKMT_STATUS_NO_MEMORY;
|
||||
}
|
||||
|
||||
if (!MemFlags.ui32.CoarseGrain)
|
||||
create_info.mem_flags = rocr_proxy::kFineGrain;
|
||||
|
||||
// create_info.mem_flags |= rocr_proxy::kKernarg;
|
||||
create_info.flags.physical_only = MemFlags.ui32.NoAddress;
|
||||
create_info.flags.interprocess = MemFlags.ui32.NoAddress;
|
||||
create_info.flags.locked = 0; //!!(alloc_flags & AllocatePinned);
|
||||
|
||||
auto code = dev->CreateGpuMemory(create_info, &gpu_mem);
|
||||
if (code == ErrorCode::Success) {
|
||||
*MemoryAddress = reinterpret_cast<void *>(gpu_mem->GpuAddress());
|
||||
std::lock_guard<std::mutex> gard(allocation_map_lock_);
|
||||
allocation_map_[*MemoryAddress] = Allocation(
|
||||
gpu_mem->GetGpuMemoryHandle(), *MemoryAddress, (uint64_t)*MemoryAddress,
|
||||
create_info.size, false, nullptr, SizeInBytes,
|
||||
MemFlags.ui32.GTTAccess ? 0 : PreferredNode, MemFlags.Value);
|
||||
return HSAKMT_STATUS_SUCCESS;
|
||||
}
|
||||
|
||||
return HSAKMT_STATUS_ERROR;
|
||||
}
|
||||
|
||||
HSAKMT_STATUS HSAKMTAPI hsaKmtFreeMemory(void *MemoryAddress,
|
||||
HSAuint64 SizeInBytes) {
|
||||
CHECK_DXG_OPEN();
|
||||
|
||||
if (!MemoryAddress)
|
||||
return HSAKMT_STATUS_INVALID_PARAMETER;
|
||||
|
||||
rocr::core::GpuMemory *gpu_mem = nullptr;
|
||||
{
|
||||
std::lock_guard<std::mutex> gard(allocation_map_lock_);
|
||||
auto it = allocation_map_.find(MemoryAddress);
|
||||
if (it == allocation_map_.end()) {
|
||||
return HSAKMT_STATUS_ERROR;
|
||||
}
|
||||
|
||||
gpu_mem = rocr::core::GpuMemory::Convert(it->second.handle);
|
||||
allocation_map_.erase(it);
|
||||
}
|
||||
|
||||
delete gpu_mem;
|
||||
|
||||
return HSAKMT_STATUS_SUCCESS;
|
||||
}
|
||||
|
||||
HSAKMT_STATUS HSAKMTAPI hsaKmtAvailableMemory(HSAuint32 Node,
|
||||
HSAuint64 *AvailableBytes) {
|
||||
CHECK_DXG_OPEN();
|
||||
|
||||
if (!AvailableBytes)
|
||||
return HSAKMT_STATUS_INVALID_PARAMETER;
|
||||
|
||||
rocr::core::WDDMDevice *dev = get_wddmdev(Node);
|
||||
if (!dev)
|
||||
return HSAKMT_STATUS_ERROR;
|
||||
|
||||
*AvailableBytes = dev->VramAvail();
|
||||
return HSAKMT_STATUS_SUCCESS;
|
||||
}
|
||||
|
||||
HSAKMT_STATUS HSAKMTAPI hsaKmtRegisterMemory(void *MemoryAddress,
|
||||
HSAuint64 MemorySizeInBytes) {
|
||||
CHECK_DXG_OPEN();
|
||||
|
||||
assert(false);
|
||||
return HSAKMT_STATUS_SUCCESS;
|
||||
}
|
||||
|
||||
HSAKMT_STATUS HSAKMTAPI hsaKmtRegisterMemoryToNodes(void *MemoryAddress,
|
||||
HSAuint64 MemorySizeInBytes,
|
||||
HSAuint64 NumberOfNodes,
|
||||
HSAuint32 *NodeArray) {
|
||||
CHECK_DXG_OPEN();
|
||||
|
||||
assert(false);
|
||||
return HSAKMT_STATUS_SUCCESS;
|
||||
}
|
||||
|
||||
HSAKMT_STATUS HSAKMTAPI hsaKmtRegisterMemoryWithFlags(
|
||||
void *MemoryAddress, HSAuint64 MemorySizeInBytes, HsaMemFlags MemFlags) {
|
||||
CHECK_DXG_OPEN();
|
||||
|
||||
if (!MemoryAddress)
|
||||
return HSAKMT_STATUS_INVALID_PARAMETER;
|
||||
|
||||
pr_debug("[%s] address %p\n", __func__, MemoryAddress);
|
||||
|
||||
if (MemFlags.ui32.ExtendedCoherent && MemFlags.ui32.CoarseGrain)
|
||||
return HSAKMT_STATUS_INVALID_PARAMETER;
|
||||
|
||||
// Registered memory should be ordinary paged host memory.
|
||||
if ((MemFlags.ui32.HostAccess != 1) || (MemFlags.ui32.NonPaged == 1))
|
||||
return HSAKMT_STATUS_NOT_SUPPORTED;
|
||||
|
||||
if (!is_dgpu)
|
||||
/* TODO: support mixed APU and dGPU configurations */
|
||||
return HSAKMT_STATUS_NOT_SUPPORTED;
|
||||
|
||||
return HSAKMT_STATUS_SUCCESS;
|
||||
}
|
||||
|
||||
HSAKMT_STATUS HSAKMTAPI hsaKmtRegisterGraphicsHandleToNodes(
|
||||
HSAuint64 GraphicsResourceHandle,
|
||||
HsaGraphicsResourceInfo *GraphicsResourceInfo, HSAuint64 NumberOfNodes,
|
||||
HSAuint32 *NodeArray) {
|
||||
CHECK_DXG_OPEN();
|
||||
|
||||
assert(false);
|
||||
return HSAKMT_STATUS_SUCCESS;
|
||||
}
|
||||
|
||||
HSAKMT_STATUS HSAKMTAPI hsaKmtExportDMABufHandle(void *MemoryAddress,
|
||||
HSAuint64 MemorySizeInBytes,
|
||||
int *DMABufFd,
|
||||
HSAuint64 *Offset) {
|
||||
CHECK_DXG_OPEN();
|
||||
assert(false);
|
||||
|
||||
return HSAKMT_STATUS_SUCCESS;
|
||||
}
|
||||
|
||||
HSAKMT_STATUS HSAKMTAPI
|
||||
hsaKmtShareMemory(void *MemoryAddress, HSAuint64 SizeInBytes,
|
||||
HsaSharedMemoryHandle *SharedMemoryHandle) {
|
||||
CHECK_DXG_OPEN();
|
||||
|
||||
assert(false);
|
||||
return HSAKMT_STATUS_SUCCESS;
|
||||
}
|
||||
|
||||
HSAKMT_STATUS HSAKMTAPI
|
||||
hsaKmtRegisterSharedHandle(const HsaSharedMemoryHandle *SharedMemoryHandle,
|
||||
void **MemoryAddress, HSAuint64 *SizeInBytes) {
|
||||
CHECK_DXG_OPEN();
|
||||
|
||||
assert(false);
|
||||
return HSAKMT_STATUS_SUCCESS;
|
||||
}
|
||||
|
||||
HSAKMT_STATUS HSAKMTAPI hsaKmtRegisterSharedHandleToNodes(
|
||||
const HsaSharedMemoryHandle *SharedMemoryHandle, void **MemoryAddress,
|
||||
HSAuint64 *SizeInBytes, HSAuint64 NumberOfNodes, HSAuint32 *NodeArray) {
|
||||
CHECK_DXG_OPEN();
|
||||
|
||||
assert(false);
|
||||
return HSAKMT_STATUS_SUCCESS;
|
||||
}
|
||||
|
||||
HSAKMT_STATUS HSAKMTAPI hsaKmtProcessVMRead(HSAuint32 Pid,
|
||||
HsaMemoryRange *LocalMemoryArray,
|
||||
HSAuint64 LocalMemoryArrayCount,
|
||||
HsaMemoryRange *RemoteMemoryArray,
|
||||
HSAuint64 RemoteMemoryArrayCount,
|
||||
HSAuint64 *SizeCopied) {
|
||||
pr_err("[%s] Deprecated\n", __func__);
|
||||
|
||||
assert(false);
|
||||
return HSAKMT_STATUS_NOT_IMPLEMENTED;
|
||||
}
|
||||
|
||||
HSAKMT_STATUS HSAKMTAPI hsaKmtProcessVMWrite(HSAuint32 Pid,
|
||||
HsaMemoryRange *LocalMemoryArray,
|
||||
HSAuint64 LocalMemoryArrayCount,
|
||||
HsaMemoryRange *RemoteMemoryArray,
|
||||
HSAuint64 RemoteMemoryArrayCount,
|
||||
HSAuint64 *SizeCopied) {
|
||||
pr_err("[%s] Deprecated\n", __func__);
|
||||
|
||||
assert(false);
|
||||
return HSAKMT_STATUS_NOT_IMPLEMENTED;
|
||||
}
|
||||
|
||||
HSAKMT_STATUS HSAKMTAPI hsaKmtDeregisterMemory(void *MemoryAddress) {
|
||||
CHECK_DXG_OPEN();
|
||||
|
||||
if (!MemoryAddress)
|
||||
return HSAKMT_STATUS_INVALID_PARAMETER;
|
||||
|
||||
pr_debug("[%s] address %p\n", __func__, MemoryAddress);
|
||||
|
||||
return HSAKMT_STATUS_SUCCESS;
|
||||
}
|
||||
|
||||
HSAKMT_STATUS HSAKMTAPI hsaKmtMapMemoryToGPU(void *MemoryAddress,
|
||||
HSAuint64 MemorySizeInBytes,
|
||||
HSAuint64 *AlternateVAGPU) {
|
||||
CHECK_DXG_OPEN();
|
||||
|
||||
if (!MemoryAddress || !AlternateVAGPU) {
|
||||
pr_err("FIXME: mapping NULL pointer\n");
|
||||
return HSAKMT_STATUS_ERROR;
|
||||
}
|
||||
|
||||
uint64_t start = rocr::AlignDown((uint64_t)MemoryAddress, 4096);
|
||||
uint64_t end =
|
||||
rocr::AlignUp((uint64_t)MemoryAddress + MemorySizeInBytes, 4096);
|
||||
|
||||
void *aligned_ptr = (void *)start;
|
||||
size_t aligned_size = end - start;
|
||||
|
||||
{
|
||||
std::lock_guard<std::mutex> gard(allocation_map_lock_);
|
||||
// GTT mem
|
||||
auto it_gtt = allocation_map_.find(aligned_ptr);
|
||||
if (it_gtt != allocation_map_.end()) {
|
||||
if (!it_gtt->second.userptr) {
|
||||
if (it_gtt->second.size >= MemorySizeInBytes) {
|
||||
*AlternateVAGPU = (uint64_t)MemoryAddress;
|
||||
return HSAKMT_STATUS_SUCCESS;
|
||||
} else {
|
||||
return HSAKMT_STATUS_ERROR;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// userptr mem
|
||||
auto it = allocation_map_.find(MemoryAddress);
|
||||
if (it != allocation_map_.end()) {
|
||||
if (it->second.userptr && it->second.size >= MemorySizeInBytes) {
|
||||
*AlternateVAGPU =
|
||||
(uintptr_t)it->second.gpu_addr +
|
||||
((uintptr_t)MemoryAddress - (uintptr_t)it->second.cpu_addr);
|
||||
return HSAKMT_STATUS_SUCCESS;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
rocr::core::WDDMDevice *dev = get_wddmdev(1);
|
||||
if (!dev)
|
||||
return HSAKMT_STATUS_ERROR;
|
||||
|
||||
rocr::core::GpuMemory *gpu_mem = nullptr;
|
||||
rocr::core::GpuMemoryHandle handle = 0;
|
||||
uint64_t addr;
|
||||
rocr::core::GpuMemoryCreateInfo create_info{};
|
||||
create_info.domain = rocr_proxy::kUserMemory;
|
||||
create_info.size = aligned_size;
|
||||
create_info.user_ptr = aligned_ptr;
|
||||
|
||||
auto code = dev->CreateGpuMemory(create_info, &gpu_mem);
|
||||
if (code == ErrorCode::Success) {
|
||||
addr = gpu_mem->GpuAddress();
|
||||
handle = gpu_mem->GetGpuMemoryHandle();
|
||||
} else {
|
||||
return HSAKMT_STATUS_ERROR;
|
||||
}
|
||||
|
||||
{
|
||||
std::lock_guard<std::mutex> guard(allocation_map_lock_);
|
||||
allocation_map_[MemoryAddress] =
|
||||
Allocation(handle, aligned_ptr, addr, aligned_size, true, MemoryAddress,
|
||||
MemorySizeInBytes);
|
||||
allocation_map_[(void *)addr] =
|
||||
Allocation(handle, aligned_ptr, addr, aligned_size, true, nullptr,
|
||||
MemorySizeInBytes);
|
||||
}
|
||||
|
||||
*AlternateVAGPU = addr + ((uintptr_t)MemoryAddress - (uintptr_t)aligned_ptr);
|
||||
|
||||
return HSAKMT_STATUS_SUCCESS;
|
||||
}
|
||||
|
||||
HSAKMT_STATUS HSAKMTAPI hsaKmtMapMemoryToGPUNodes(
|
||||
void *MemoryAddress, HSAuint64 MemorySizeInBytes, HSAuint64 *AlternateVAGPU,
|
||||
HsaMemMapFlags MemMapFlags, HSAuint64 NumberOfNodes, HSAuint32 *NodeArray) {
|
||||
return hsaKmtMapMemoryToGPU(MemoryAddress, MemorySizeInBytes, AlternateVAGPU);
|
||||
}
|
||||
|
||||
HSAKMT_STATUS HSAKMTAPI hsaKmtUnmapMemoryToGPU(void *MemoryAddress) {
|
||||
CHECK_DXG_OPEN();
|
||||
|
||||
if (!MemoryAddress) {
|
||||
/* Workaround for runtime bug */
|
||||
pr_err("FIXME: Unmapping NULL pointer\n");
|
||||
return HSAKMT_STATUS_SUCCESS;
|
||||
}
|
||||
|
||||
pr_debug("[%s] address %p\n", __func__, MemoryAddress);
|
||||
|
||||
rocr::core::GpuMemoryHandle handle = nullptr;
|
||||
{
|
||||
std::lock_guard<std::mutex> gard(allocation_map_lock_);
|
||||
auto it = allocation_map_.find(MemoryAddress);
|
||||
if (it == allocation_map_.end()) {
|
||||
return HSAKMT_STATUS_ERROR;
|
||||
}
|
||||
|
||||
if (!it->second.userptr) {
|
||||
return HSAKMT_STATUS_SUCCESS;
|
||||
}
|
||||
|
||||
handle = it->second.handle;
|
||||
|
||||
allocation_map_.erase((void *)it->second.gpu_addr);
|
||||
allocation_map_.erase(it);
|
||||
}
|
||||
auto gpu_mem = rocr::core::GpuMemory::Convert(handle);
|
||||
delete gpu_mem;
|
||||
|
||||
return HSAKMT_STATUS_SUCCESS;
|
||||
}
|
||||
|
||||
HSAKMT_STATUS HSAKMTAPI hsaKmtMapGraphicHandle(HSAuint32 NodeId,
|
||||
HSAuint64 GraphicDeviceHandle,
|
||||
HSAuint64 GraphicResourceHandle,
|
||||
HSAuint64 GraphicResourceOffset,
|
||||
HSAuint64 GraphicResourceSize,
|
||||
HSAuint64 *FlatMemoryAddress) {
|
||||
/* This API was only ever implemented in KFD for Kaveri and
|
||||
* was never upstreamed. There are no open-source users of
|
||||
* this interface. It has been superseded by
|
||||
* RegisterGraphicsHandleToNodes.
|
||||
*/
|
||||
return HSAKMT_STATUS_NOT_IMPLEMENTED;
|
||||
}
|
||||
|
||||
HSAKMT_STATUS HSAKMTAPI hsaKmtUnmapGraphicHandle(HSAuint32 NodeId,
|
||||
HSAuint64 FlatMemoryAddress,
|
||||
HSAuint64 SizeInBytes) {
|
||||
CHECK_DXG_OPEN();
|
||||
|
||||
assert(false);
|
||||
return HSAKMT_STATUS_SUCCESS;
|
||||
}
|
||||
|
||||
HSAKMT_STATUS HSAKMTAPI hsaKmtGetTileConfig(HSAuint32 NodeId,
|
||||
HsaGpuTileConfig *config) {
|
||||
CHECK_DXG_OPEN();
|
||||
|
||||
assert(false);
|
||||
return HSAKMT_STATUS_SUCCESS;
|
||||
}
|
||||
|
||||
HSAKMT_STATUS HSAKMTAPI hsaKmtQueryPointerInfo(const void *Pointer,
|
||||
HsaPointerInfo *PointerInfo) {
|
||||
CHECK_DXG_OPEN();
|
||||
|
||||
if (!Pointer || !PointerInfo)
|
||||
return HSAKMT_STATUS_INVALID_PARAMETER;
|
||||
|
||||
pr_debug("[%s] pointer %p\n", __func__, Pointer);
|
||||
|
||||
memset(PointerInfo, 0, sizeof(HsaPointerInfo));
|
||||
|
||||
Allocation allocation_info;
|
||||
{
|
||||
std::lock_guard<std::mutex> gard(allocation_map_lock_);
|
||||
auto it = allocation_map_.find(Pointer);
|
||||
if (it == allocation_map_.end()) {
|
||||
PointerInfo->Type = HSA_POINTER_UNKNOWN;
|
||||
return HSAKMT_STATUS_ERROR;
|
||||
}
|
||||
allocation_info = it->second;
|
||||
}
|
||||
|
||||
if (allocation_info.userptr) {
|
||||
PointerInfo->Type = HSA_POINTER_REGISTERED_USER;
|
||||
PointerInfo->SizeInBytes = allocation_info.size;
|
||||
} else {
|
||||
PointerInfo->Type = HSA_POINTER_ALLOCATED;
|
||||
PointerInfo->SizeInBytes = allocation_info.size_requested;
|
||||
}
|
||||
|
||||
PointerInfo->Node = allocation_info.node_id;
|
||||
PointerInfo->MemFlags.Value = allocation_info.mem_flags_value;
|
||||
PointerInfo->CPUAddress = allocation_info.cpu_addr;
|
||||
PointerInfo->GPUAddress = allocation_info.gpu_addr;
|
||||
PointerInfo->UserData = allocation_info.user_data;
|
||||
|
||||
return HSAKMT_STATUS_SUCCESS;
|
||||
}
|
||||
|
||||
HSAKMT_STATUS HSAKMTAPI hsaKmtSetMemoryUserData(const void *Pointer,
|
||||
void *UserData) {
|
||||
CHECK_DXG_OPEN();
|
||||
|
||||
assert(false);
|
||||
return HSAKMT_STATUS_SUCCESS;
|
||||
}
|
||||
|
||||
HSAKMT_STATUS HSAKMTAPI hsaKmtReplaceAsanHeaderPage(void *addr) {
|
||||
assert(false);
|
||||
#ifdef SANITIZER_AMDGPU
|
||||
pr_debug("[%s] address %p\n", __func__, addr);
|
||||
CHECK_DXG_OPEN();
|
||||
|
||||
return HSAKMT_STATUS_SUCCESS;
|
||||
#else
|
||||
return HSAKMT_STATUS_NOT_SUPPORTED;
|
||||
#endif
|
||||
}
|
||||
|
||||
HSAKMT_STATUS HSAKMTAPI hsaKmtReturnAsanHeaderPage(void *addr) {
|
||||
assert(false);
|
||||
#ifdef SANITIZER_AMDGPU
|
||||
pr_debug("[%s] address %p\n", __func__, addr);
|
||||
CHECK_DXG_OPEN();
|
||||
|
||||
return HSAKMT_STATUS_SUCCESS;
|
||||
#else
|
||||
return HSAKMT_STATUS_NOT_SUPPORTED;
|
||||
#endif
|
||||
}
|
||||
@@ -0,0 +1,274 @@
|
||||
/*
|
||||
* Copyright © 2014 Advanced Micro Devices, Inc.
|
||||
*
|
||||
* Permission is hereby granted, free of charge, to any person
|
||||
* obtaining a copy of this software and associated documentation
|
||||
* files (the "Software"), to deal in the Software without
|
||||
* restriction, including without limitation the rights to use, copy,
|
||||
* modify, merge, publish, distribute, sublicense, and/or sell copies
|
||||
* of the Software, and to permit persons to whom the Software is
|
||||
* furnished to do so, subject to the following conditions:
|
||||
*
|
||||
* The above copyright notice and this permission notice (including
|
||||
* the next paragraph) shall be included in all copies or substantial
|
||||
* portions of the Software.
|
||||
*
|
||||
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
||||
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
||||
* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
||||
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
|
||||
* HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
|
||||
* WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
||||
* DEALINGS IN THE SOFTWARE.
|
||||
*/
|
||||
#include <stdlib.h>
|
||||
#include <cstring>
|
||||
#include <sys/types.h>
|
||||
#include <sys/stat.h>
|
||||
#include <fcntl.h>
|
||||
#include <unistd.h>
|
||||
#include <cstdio>
|
||||
#include <strings.h>
|
||||
#include <dlfcn.h>
|
||||
#include <cassert>
|
||||
#include <amdgpu.h>
|
||||
#include "libhsakmt.h"
|
||||
#include "inc/hsa/hsa.h"
|
||||
#include "inc/hsa/hsa_ven_amd_loader.h"
|
||||
|
||||
int (*fn_amdgpu_device_get_fd)(HsaAMDGPUDeviceHandle device_handle);
|
||||
|
||||
hsa_signal_value_t (*fn_hsa_signal_load_relaxed)(hsa_signal_t signal);
|
||||
hsa_signal_value_t (*fn_hsa_signal_wait_relaxed)(
|
||||
hsa_signal_t signal, hsa_signal_condition_t condition,
|
||||
hsa_signal_value_t compare_value, uint64_t timeout_hint,
|
||||
hsa_wait_state_t wait_state_hint);
|
||||
void (*fn_hsa_signal_store_screlease)(hsa_signal_t hsa_signal,
|
||||
hsa_signal_value_t value);
|
||||
hsa_status_t (*fn_hsa_ven_amd_loader_query_host_address)(
|
||||
const void *device_address, const void **host_address);
|
||||
|
||||
static const char dxg_device_name[] = "/dev/dxg";
|
||||
static pid_t parent_pid = -1;
|
||||
int hsakmt_debug_level;
|
||||
bool hsakmt_forked;
|
||||
static int dxg_fd = -1;
|
||||
|
||||
/* is_forked_child detects when the process has forked since the last
|
||||
* time this function was called. We cannot rely on pthread_atfork
|
||||
* because the process can fork without calling the fork function in
|
||||
* libc (using clone or calling the system call directly).
|
||||
*/
|
||||
bool is_forked_child(void) {
|
||||
pid_t cur_pid;
|
||||
|
||||
if (hsakmt_forked)
|
||||
return true;
|
||||
|
||||
cur_pid = getpid();
|
||||
|
||||
if (parent_pid == -1) {
|
||||
parent_pid = cur_pid;
|
||||
return false;
|
||||
}
|
||||
|
||||
if (parent_pid != cur_pid) {
|
||||
hsakmt_forked = true;
|
||||
return true;
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
/* Callbacks from pthread_atfork */
|
||||
static void prepare_fork_handler(void) { pthread_mutex_lock(&hsakmt_mutex); }
|
||||
static void parent_fork_handler(void) { pthread_mutex_unlock(&hsakmt_mutex); }
|
||||
static void child_fork_handler(void) {
|
||||
pthread_mutex_init(&hsakmt_mutex, NULL);
|
||||
hsakmt_forked = true;
|
||||
}
|
||||
|
||||
/* Call this from the child process after fork. This will clear all
|
||||
* data that is duplicated from the parent process, that is not valid
|
||||
* in the child.
|
||||
* The topology information is duplicated from the parent is valid
|
||||
* in the child process so it is not cleared
|
||||
*/
|
||||
static void clear_after_fork(void) {
|
||||
// TODO: fmm_clear_all_mem();
|
||||
if (dxg_fd) {
|
||||
close(dxg_fd);
|
||||
dxg_fd = -1;
|
||||
}
|
||||
dxg_open_count = 0;
|
||||
parent_pid = -1;
|
||||
hsakmt_forked = false;
|
||||
}
|
||||
|
||||
static inline void init_page_size(void) {
|
||||
#ifndef PAGE_SIZE
|
||||
PAGE_SIZE = sysconf(_SC_PAGESIZE);
|
||||
#endif
|
||||
PAGE_SHIFT = ffs(PAGE_SIZE) - 1;
|
||||
}
|
||||
|
||||
static HSAKMT_STATUS init_vars_from_env(void) {
|
||||
char *envvar;
|
||||
int debug_level;
|
||||
|
||||
/* Normally libraries don't print messages. For debugging purpose, we'll
|
||||
* print messages if an environment variable, HSAKMT_DEBUG_LEVEL, is set.
|
||||
*/
|
||||
hsakmt_debug_level = HSAKMT_DEBUG_LEVEL_DEFAULT;
|
||||
|
||||
envvar = getenv("HSAKMT_DEBUG_LEVEL");
|
||||
if (envvar) {
|
||||
debug_level = atoi(envvar);
|
||||
if (debug_level >= HSAKMT_DEBUG_LEVEL_ERR &&
|
||||
debug_level <= HSAKMT_DEBUG_LEVEL_DEBUG)
|
||||
hsakmt_debug_level = debug_level;
|
||||
}
|
||||
|
||||
/* Check whether to support Zero frame buffer */
|
||||
envvar = getenv("HSA_ZFB");
|
||||
if (envvar)
|
||||
zfb_support = atoi(envvar);
|
||||
|
||||
/* Check whether to handle vendor specific aql packet */
|
||||
envvar = getenv("WSLKMT_VENDOR_PACKET");
|
||||
if (envvar)
|
||||
vendor_packet_support = atoi(envvar);
|
||||
|
||||
return HSAKMT_STATUS_SUCCESS;
|
||||
}
|
||||
|
||||
#define _HSAKMT_LOOKUP_SYMS(_sym) \
|
||||
do { \
|
||||
fn_##_sym = \
|
||||
reinterpret_cast<decltype(fn_##_sym)>(dlsym(RTLD_DEFAULT, #_sym)); \
|
||||
if (!fn_##_sym) { \
|
||||
pr_err("%s not found - %s\n", #_sym, dlerror()); \
|
||||
return HSAKMT_STATUS_ERROR; \
|
||||
} \
|
||||
} while (0)
|
||||
|
||||
static HSAKMT_STATUS init_symbols(void) {
|
||||
_HSAKMT_LOOKUP_SYMS(hsa_signal_load_relaxed);
|
||||
_HSAKMT_LOOKUP_SYMS(hsa_signal_wait_relaxed);
|
||||
_HSAKMT_LOOKUP_SYMS(hsa_signal_store_screlease);
|
||||
|
||||
hsa_status_t (*fn_hsa_system_get_extension_table)(
|
||||
uint16_t extension, uint16_t version_major, uint16_t version_minor,
|
||||
void *table);
|
||||
_HSAKMT_LOOKUP_SYMS(hsa_system_get_extension_table);
|
||||
hsa_ven_amd_loader_1_03_pfn_t table;
|
||||
fn_hsa_system_get_extension_table(HSA_EXTENSION_AMD_LOADER, 1, 3, &table);
|
||||
fn_hsa_ven_amd_loader_query_host_address =
|
||||
table.hsa_ven_amd_loader_query_host_address;
|
||||
|
||||
return HSAKMT_STATUS_SUCCESS;
|
||||
}
|
||||
|
||||
static void load_libdrm_amdgpu(void) {
|
||||
/* load libdrm_amdgpu */
|
||||
int fd;
|
||||
uint32_t major, minor;
|
||||
amdgpu_device_handle device_handle;
|
||||
amdgpu_device_initialize(fd, &major, &minor, &device_handle);
|
||||
}
|
||||
|
||||
HSAKMT_STATUS HSAKMTAPI hsaKmtOpenKFD(void) {
|
||||
HSAKMT_STATUS result;
|
||||
int fd = -1;
|
||||
HsaSystemProperties sys_props;
|
||||
char *error;
|
||||
char *useSvmStr;
|
||||
|
||||
pthread_mutex_lock(&hsakmt_mutex);
|
||||
|
||||
/* If the process has forked, the child process must re-initialize
|
||||
* it's connection to DXG. Any references tracked by dxg_open_count
|
||||
* belong to the parent
|
||||
*/
|
||||
if (is_forked_child())
|
||||
clear_after_fork();
|
||||
|
||||
if (dxg_open_count == 0) {
|
||||
static bool atfork_installed = false;
|
||||
|
||||
result = init_symbols();
|
||||
if (result != HSAKMT_STATUS_SUCCESS)
|
||||
goto open_failed;
|
||||
|
||||
load_libdrm_amdgpu();
|
||||
|
||||
result = init_vars_from_env();
|
||||
if (result != HSAKMT_STATUS_SUCCESS)
|
||||
goto open_failed;
|
||||
|
||||
if (dxg_fd < 0) {
|
||||
fd = open(dxg_device_name, O_RDWR | O_CLOEXEC);
|
||||
|
||||
if (fd == -1) {
|
||||
result = HSAKMT_STATUS_KERNEL_IO_CHANNEL_NOT_OPENED;
|
||||
goto open_failed;
|
||||
}
|
||||
|
||||
dxg_fd = fd;
|
||||
}
|
||||
|
||||
init_page_size();
|
||||
|
||||
useSvmStr = getenv("HSA_USE_SVM");
|
||||
is_svm_api_supported = !(useSvmStr && !strcmp(useSvmStr, "0")) && false;
|
||||
|
||||
// result = topology_sysfs_get_system_props(&sys_props);
|
||||
if (result != HSAKMT_STATUS_SUCCESS)
|
||||
goto topology_sysfs_failed;
|
||||
|
||||
dxg_open_count = 1;
|
||||
|
||||
if (!atfork_installed) {
|
||||
/* Atfork handlers cannot be uninstalled and
|
||||
* must be installed only once. Otherwise
|
||||
* prepare will deadlock when trying to take
|
||||
* the same lock multiple times.
|
||||
*/
|
||||
pthread_atfork(prepare_fork_handler, parent_fork_handler,
|
||||
child_fork_handler);
|
||||
atfork_installed = true;
|
||||
}
|
||||
} else {
|
||||
dxg_open_count++;
|
||||
result = HSAKMT_STATUS_KERNEL_ALREADY_OPENED;
|
||||
}
|
||||
|
||||
pthread_mutex_unlock(&hsakmt_mutex);
|
||||
return result;
|
||||
topology_sysfs_failed:
|
||||
close(fd);
|
||||
open_failed:
|
||||
pthread_mutex_unlock(&hsakmt_mutex);
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
HSAKMT_STATUS HSAKMTAPI hsaKmtCloseKFD(void) {
|
||||
HSAKMT_STATUS result;
|
||||
|
||||
pthread_mutex_lock(&hsakmt_mutex);
|
||||
|
||||
if (dxg_open_count > 0) {
|
||||
if (--dxg_open_count == 0) {
|
||||
close(dxg_fd);
|
||||
}
|
||||
|
||||
result = HSAKMT_STATUS_SUCCESS;
|
||||
} else
|
||||
result = HSAKMT_STATUS_KERNEL_IO_CHANNEL_NOT_OPENED;
|
||||
|
||||
pthread_mutex_unlock(&hsakmt_mutex);
|
||||
|
||||
return result;
|
||||
}
|
||||
@@ -0,0 +1,73 @@
|
||||
/*
|
||||
* Copyright © 2023 Advanced Micro Devices, Inc.
|
||||
*
|
||||
* Permission is hereby granted, free of charge, to any person
|
||||
* obtaining a copy of this software and associated documentation
|
||||
* files (the "Software"), to deal in the Software without
|
||||
* restriction, including without limitation the rights to use, copy,
|
||||
* modify, merge, publish, distribute, sublicense, and/or sell copies
|
||||
* of the Software, and to permit persons to whom the Software is
|
||||
* furnished to do so, subject to the following conditions:
|
||||
*
|
||||
* The above copyright notice and this permission notice (including
|
||||
* the next paragraph) shall be included in all copies or substantial
|
||||
* portions of the Software.
|
||||
*
|
||||
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
||||
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
||||
* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
||||
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
|
||||
* HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
|
||||
* WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
||||
* DEALINGS IN THE SOFTWARE.
|
||||
*/
|
||||
|
||||
#include <assert.h>
|
||||
#include <errno.h>
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include "libhsakmt.h"
|
||||
|
||||
HSAKMT_STATUS HSAKMTAPI hsaKmtPcSamplingSupport(void) {
|
||||
CHECK_DXG_OPEN();
|
||||
// Used for profiling tools
|
||||
return HSAKMT_STATUS_NOT_SUPPORTED;
|
||||
}
|
||||
|
||||
HSAKMT_STATUS HSAKMTAPI
|
||||
hsaKmtPcSamplingQueryCapabilities(HSAuint32 NodeId, void *sample_info,
|
||||
HSAuint32 sample_info_sz, HSAuint32 *size) {
|
||||
CHECK_DXG_OPEN();
|
||||
// Used for profiling tools
|
||||
return HSAKMT_STATUS_NOT_SUPPORTED;
|
||||
}
|
||||
|
||||
HSAKMT_STATUS HSAKMTAPI hsaKmtPcSamplingCreate(HSAuint32 NodeId,
|
||||
HsaPcSamplingInfo *sample_info,
|
||||
HsaPcSamplingTraceId *traceId) {
|
||||
CHECK_DXG_OPEN();
|
||||
// Used for profiling tools
|
||||
return HSAKMT_STATUS_NOT_SUPPORTED;
|
||||
}
|
||||
|
||||
HSAKMT_STATUS HSAKMTAPI hsaKmtPcSamplingDestroy(HSAuint32 NodeId,
|
||||
HsaPcSamplingTraceId traceId) {
|
||||
CHECK_DXG_OPEN();
|
||||
// Used for profiling tools
|
||||
return HSAKMT_STATUS_NOT_SUPPORTED;
|
||||
}
|
||||
|
||||
HSAKMT_STATUS HSAKMTAPI hsaKmtPcSamplingStart(HSAuint32 NodeId,
|
||||
HsaPcSamplingTraceId traceId) {
|
||||
CHECK_DXG_OPEN();
|
||||
// Used for profiling tools
|
||||
return HSAKMT_STATUS_NOT_SUPPORTED;
|
||||
}
|
||||
|
||||
HSAKMT_STATUS HSAKMTAPI hsaKmtPcSamplingStop(HSAuint32 NodeId,
|
||||
HsaPcSamplingTraceId traceId) {
|
||||
CHECK_DXG_OPEN();
|
||||
// Used for profiling tools
|
||||
return HSAKMT_STATUS_NOT_SUPPORTED;
|
||||
}
|
||||
@@ -0,0 +1,82 @@
|
||||
/*
|
||||
* Copyright © 2014 Advanced Micro Devices, Inc.
|
||||
*
|
||||
* Permission is hereby granted, free of charge, to any person
|
||||
* obtaining a copy of this software and associated documentation
|
||||
* files (the "Software"), to deal in the Software without
|
||||
* restriction, including without limitation the rights to use, copy,
|
||||
* modify, merge, publish, distribute, sublicense, and/or sell copies
|
||||
* of the Software, and to permit persons to whom the Software is
|
||||
* furnished to do so, subject to the following conditions:
|
||||
*
|
||||
* The above copyright notice and this permission notice (including
|
||||
* the next paragraph) shall be included in all copies or substantial
|
||||
* portions of the Software.
|
||||
*
|
||||
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
||||
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
||||
* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
||||
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
|
||||
* HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
|
||||
* WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
||||
* DEALINGS IN THE SOFTWARE.
|
||||
*/
|
||||
#include "libhsakmt.h"
|
||||
|
||||
HSAKMT_STATUS HSAKMTAPI hsaKmtPmcGetCounterProperties(
|
||||
HSAuint32 NodeId, HsaCounterProperties **CounterProperties) {
|
||||
return HSAKMT_STATUS_NOT_SUPPORTED;
|
||||
}
|
||||
|
||||
/* Registers a set of (HW) counters to be used for tracing/profiling */
|
||||
HSAKMT_STATUS HSAKMTAPI hsaKmtPmcRegisterTrace(HSAuint32 NodeId,
|
||||
HSAuint32 NumberOfCounters,
|
||||
HsaCounter *Counters,
|
||||
HsaPmcTraceRoot *TraceRoot) {
|
||||
CHECK_DXG_OPEN();
|
||||
return HSAKMT_STATUS_NOT_SUPPORTED;
|
||||
}
|
||||
|
||||
/* Unregisters a set of (HW) counters used for tracing/profiling */
|
||||
|
||||
HSAKMT_STATUS HSAKMTAPI hsaKmtPmcUnregisterTrace(HSAuint32 NodeId,
|
||||
HSATraceId TraceId) {
|
||||
CHECK_DXG_OPEN();
|
||||
return HSAKMT_STATUS_NOT_SUPPORTED;
|
||||
}
|
||||
|
||||
HSAKMT_STATUS HSAKMTAPI hsaKmtPmcAcquireTraceAccess(HSAuint32 NodeId,
|
||||
HSATraceId TraceId) {
|
||||
CHECK_DXG_OPEN();
|
||||
return HSAKMT_STATUS_NOT_SUPPORTED;
|
||||
}
|
||||
|
||||
HSAKMT_STATUS HSAKMTAPI hsaKmtPmcReleaseTraceAccess(HSAuint32 NodeId,
|
||||
HSATraceId TraceId) {
|
||||
CHECK_DXG_OPEN();
|
||||
return HSAKMT_STATUS_NOT_SUPPORTED;
|
||||
}
|
||||
|
||||
/* Starts tracing operation on a previously established set of performance
|
||||
* counters */
|
||||
HSAKMT_STATUS HSAKMTAPI hsaKmtPmcStartTrace(HSATraceId TraceId,
|
||||
void *TraceBuffer,
|
||||
HSAuint64 TraceBufferSizeBytes) {
|
||||
CHECK_DXG_OPEN();
|
||||
return HSAKMT_STATUS_NOT_SUPPORTED;
|
||||
}
|
||||
|
||||
/*Forces an update of all the counters that a previously started trace operation
|
||||
* has registered */
|
||||
HSAKMT_STATUS HSAKMTAPI hsaKmtPmcQueryTrace(HSATraceId TraceId) {
|
||||
CHECK_DXG_OPEN();
|
||||
return HSAKMT_STATUS_NOT_SUPPORTED;
|
||||
}
|
||||
|
||||
/* Stops tracing operation on a previously established set of performance
|
||||
* counters */
|
||||
HSAKMT_STATUS HSAKMTAPI hsaKmtPmcStopTrace(HSATraceId TraceId) {
|
||||
CHECK_DXG_OPEN();
|
||||
return HSAKMT_STATUS_NOT_SUPPORTED;
|
||||
}
|
||||
+174
@@ -0,0 +1,174 @@
|
||||
/*
|
||||
* Copyright © 2014 Advanced Micro Devices, Inc.
|
||||
*
|
||||
* Permission is hereby granted, free of charge, to any person
|
||||
* obtaining a copy of this software and associated documentation
|
||||
* files (the "Software"), to deal in the Software without
|
||||
* restriction, including without limitation the rights to use, copy,
|
||||
* modify, merge, publish, distribute, sublicense, and/or sell copies
|
||||
* of the Software, and to permit persons to whom the Software is
|
||||
* furnished to do so, subject to the following conditions:
|
||||
*
|
||||
* The above copyright notice and this permission notice (including
|
||||
* the next paragraph) shall be included in all copies or substantial
|
||||
* portions of the Software.
|
||||
*
|
||||
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
||||
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
||||
* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
||||
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
|
||||
* HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
|
||||
* WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
||||
* DEALINGS IN THE SOFTWARE.
|
||||
*/
|
||||
#include <cinttypes>
|
||||
#include "libhsakmt.h"
|
||||
#include "inc/wddm/device.h"
|
||||
#include "inc/wddm/queue.h"
|
||||
#include "hsa-runtime/inc/amd_hsa_signal.h"
|
||||
|
||||
uint32_t get_vgpr_size_per_cu(HSA_ENGINE_ID id) {
|
||||
uint32_t vgpr_size = 0x40000;
|
||||
|
||||
if (id.ui32.Major >= 11) {
|
||||
vgpr_size = 0x60000;
|
||||
}
|
||||
|
||||
return vgpr_size;
|
||||
}
|
||||
|
||||
HSAKMT_STATUS HSAKMTAPI hsaKmtCreateQueue(
|
||||
HSAuint32 NodeId, HSA_QUEUE_TYPE Type, HSAuint32 QueuePercentage,
|
||||
HSA_QUEUE_PRIORITY Priority, void *QueueAddress, HSAuint64 QueueSizeInBytes,
|
||||
HsaEvent *Event, HsaQueueResource *QueueResource) {
|
||||
HSAKMT_STATUS result;
|
||||
|
||||
CHECK_DXG_OPEN();
|
||||
assert(Event == nullptr);
|
||||
|
||||
if (Priority < HSA_QUEUE_PRIORITY_MINIMUM ||
|
||||
Priority > HSA_QUEUE_PRIORITY_MAXIMUM)
|
||||
return HSAKMT_STATUS_INVALID_PARAMETER;
|
||||
|
||||
rocr::core::WDDMDevice *device_ = get_wddmdev(NodeId);
|
||||
assert(device_);
|
||||
|
||||
switch (Type) {
|
||||
case HSA_QUEUE_COMPUTE_AQL: {
|
||||
assert(QueueResource->ErrorReason == nullptr);
|
||||
uint64_t pkg_num = QueueSizeInBytes / 64;
|
||||
uint32_t cmdbuf_size = device_->GetCmdbufSize();
|
||||
uint32_t queue_engine = device_->GetComputeEngine();
|
||||
bool use_hws = device_->IsHwsEnabled(queue_engine);
|
||||
auto queue_ = new rocr::core::ComputeQueue(
|
||||
device_, QueueAddress, pkg_num,
|
||||
reinterpret_cast<std::atomic<uint64_t> *>(QueueResource->Queue_write_ptr_aql),
|
||||
reinterpret_cast<std::atomic<uint64_t> *>(QueueResource->Queue_read_ptr_aql),
|
||||
QueueResource->ErrorReason, cmdbuf_size, queue_engine, use_hws);
|
||||
|
||||
QueueResource->QueueId = reinterpret_cast<HSA_QUEUEID>(queue_);
|
||||
// for doorbell_signal.hardware_doorbell_ptr
|
||||
QueueResource->Queue_DoorBell_aql = queue_->GetDoorbellPtr();
|
||||
} break;
|
||||
case HSA_QUEUE_SDMA:
|
||||
default:
|
||||
assert(false);
|
||||
QueueResource->QueueId = 0;
|
||||
QueueResource->Queue_DoorBell = nullptr;
|
||||
break;
|
||||
}
|
||||
|
||||
return HSAKMT_STATUS_SUCCESS;
|
||||
}
|
||||
|
||||
HSAKMT_STATUS HSAKMTAPI hsaKmtUpdateQueue(
|
||||
HSA_QUEUEID QueueId, HSAuint32 QueuePercentage, HSA_QUEUE_PRIORITY Priority,
|
||||
void *QueueAddress, HSAuint64 QueueSize, HsaEvent *Event) {
|
||||
CHECK_DXG_OPEN();
|
||||
|
||||
if (Priority < HSA_QUEUE_PRIORITY_MINIMUM ||
|
||||
Priority > HSA_QUEUE_PRIORITY_MAXIMUM)
|
||||
return HSAKMT_STATUS_INVALID_PARAMETER;
|
||||
|
||||
auto queue_ = reinterpret_cast<rocr::core::ComputeQueue *>(QueueId);
|
||||
if (!queue_)
|
||||
return HSAKMT_STATUS_INVALID_PARAMETER;
|
||||
|
||||
return HSAKMT_STATUS_SUCCESS;
|
||||
}
|
||||
|
||||
HSAKMT_STATUS HSAKMTAPI hsaKmtDestroyQueue(HSA_QUEUEID QueueId) {
|
||||
CHECK_DXG_OPEN();
|
||||
|
||||
auto queue_ = reinterpret_cast<rocr::core::ComputeQueue *>(QueueId);
|
||||
|
||||
if (!queue_)
|
||||
return HSAKMT_STATUS_INVALID_PARAMETER;
|
||||
|
||||
delete queue_;
|
||||
return HSAKMT_STATUS_SUCCESS;
|
||||
}
|
||||
|
||||
HSAKMT_STATUS HSAKMTAPI hsaKmtSetQueueCUMask(HSA_QUEUEID QueueId,
|
||||
HSAuint32 CUMaskCount,
|
||||
HSAuint32 *QueueCUMask) {
|
||||
CHECK_DXG_OPEN();
|
||||
|
||||
auto queue_ = reinterpret_cast<rocr::core::ComputeQueue *>(QueueId);
|
||||
if (!queue_)
|
||||
return HSAKMT_STATUS_INVALID_PARAMETER;
|
||||
|
||||
if (CUMaskCount == 0 || !QueueCUMask || ((CUMaskCount % 32) != 0))
|
||||
return HSAKMT_STATUS_INVALID_PARAMETER;
|
||||
|
||||
pr_debug("%s not implemented\n", __func__);
|
||||
|
||||
return HSAKMT_STATUS_SUCCESS;
|
||||
}
|
||||
|
||||
HSAKMT_STATUS HSAKMTAPI hsaKmtGetQueueInfo(HSA_QUEUEID QueueId,
|
||||
HsaQueueInfo *QueueInfo) {
|
||||
CHECK_DXG_OPEN();
|
||||
|
||||
if (QueueInfo == NULL)
|
||||
return HSAKMT_STATUS_INVALID_PARAMETER;
|
||||
memset(QueueInfo, 0, sizeof(*QueueInfo));
|
||||
|
||||
assert(false);
|
||||
return HSAKMT_STATUS_SUCCESS;
|
||||
}
|
||||
|
||||
HSAKMT_STATUS HSAKMTAPI hsaKmtSetTrapHandler(HSAuint32 Node,
|
||||
void *TrapHandlerBaseAddress,
|
||||
HSAuint64 TrapHandlerSizeInBytes,
|
||||
void *TrapBufferBaseAddress,
|
||||
HSAuint64 TrapBufferSizeInBytes) {
|
||||
CHECK_DXG_OPEN();
|
||||
|
||||
pr_debug("%s not implemented\n", __func__);
|
||||
return HSAKMT_STATUS_SUCCESS;
|
||||
}
|
||||
|
||||
HSAKMT_STATUS HSAKMTAPI hsaKmtAllocQueueGWS(HSA_QUEUEID QueueId, HSAuint32 nGWS,
|
||||
HSAuint32 *firstGWS) {
|
||||
CHECK_DXG_OPEN();
|
||||
|
||||
auto queue_ = reinterpret_cast<rocr::core::ComputeQueue *>(QueueId);
|
||||
if (!queue_)
|
||||
return HSAKMT_STATUS_INVALID_PARAMETER;
|
||||
|
||||
assert(false);
|
||||
return HSAKMT_STATUS_SUCCESS;
|
||||
}
|
||||
|
||||
HSAKMT_STATUS HSAKMTAPI hsaKmtQueueRingDoorbell(HSA_QUEUEID QueueId) {
|
||||
CHECK_DXG_OPEN();
|
||||
|
||||
auto queue_ = reinterpret_cast<rocr::core::ComputeQueue *>(QueueId);
|
||||
if (!queue_)
|
||||
return HSAKMT_STATUS_INVALID_PARAMETER;
|
||||
|
||||
queue_->RingDoorbell();
|
||||
return HSAKMT_STATUS_SUCCESS;
|
||||
}
|
||||
Bināro failu nav iespējams attēlot.
+48
@@ -0,0 +1,48 @@
|
||||
/*
|
||||
* Copyright © 2020 Advanced Micro Devices, Inc.
|
||||
*
|
||||
* Permission is hereby granted, free of charge, to any person
|
||||
* obtaining a copy of this software and associated documentation
|
||||
* files (the "Software"), to deal in the Software without
|
||||
* restriction, including without limitation the rights to use, copy,
|
||||
* modify, merge, publish, distribute, sublicense, and/or sell copies
|
||||
* of the Software, and to permit persons to whom the Software is
|
||||
* furnished to do so, subject to the following conditions:
|
||||
*
|
||||
* The above copyright notice and this permission notice (including
|
||||
* the next paragraph) shall be included in all copies or substantial
|
||||
* portions of the Software.
|
||||
*
|
||||
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
||||
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
||||
* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
||||
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
|
||||
* HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
|
||||
* WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
||||
* DEALINGS IN THE SOFTWARE.
|
||||
*/
|
||||
|
||||
#include <stdlib.h>
|
||||
#include <stdio.h>
|
||||
#include "libhsakmt.h"
|
||||
|
||||
HSAKMT_STATUS HSAKMTAPI hsaKmtSPMAcquire(HSAuint32 PreferredNode) {
|
||||
CHECK_DXG_OPEN();
|
||||
// Used for profiling tools
|
||||
return HSAKMT_STATUS_NOT_SUPPORTED;
|
||||
}
|
||||
|
||||
HSAKMT_STATUS HSAKMTAPI hsaKmtSPMSetDestBuffer(
|
||||
HSAuint32 PreferredNode, HSAuint32 SizeInBytes, HSAuint32 *timeout,
|
||||
HSAuint32 *SizeCopied, void *DestMemoryAddress, bool *isSPMDataLoss) {
|
||||
CHECK_DXG_OPEN();
|
||||
// Used for profiling tools
|
||||
return HSAKMT_STATUS_NOT_SUPPORTED;
|
||||
}
|
||||
|
||||
HSAKMT_STATUS HSAKMTAPI hsaKmtSPMRelease(HSAuint32 PreferredNode) {
|
||||
CHECK_DXG_OPEN();
|
||||
// Used for profiling tools
|
||||
return HSAKMT_STATUS_NOT_SUPPORTED;
|
||||
}
|
||||
+52
@@ -0,0 +1,52 @@
|
||||
/*
|
||||
* Copyright © 2020 Advanced Micro Devices, Inc.
|
||||
*
|
||||
* Permission is hereby granted, free of charge, to any person
|
||||
* obtaining a copy of this software and associated documentation
|
||||
* files (the "Software"), to deal in the Software without
|
||||
* restriction, including without limitation the rights to use, copy,
|
||||
* modify, merge, publish, distribute, sublicense, and/or sell copies
|
||||
* of the Software, and to permit persons to whom the Software is
|
||||
* furnished to do so, subject to the following conditions:
|
||||
*
|
||||
* The above copyright notice and this permission notice (including
|
||||
* the next paragraph) shall be included in all copies or substantial
|
||||
* portions of the Software.
|
||||
*
|
||||
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
||||
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
||||
* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
||||
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
|
||||
* HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
|
||||
* WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
||||
* DEALINGS IN THE SOFTWARE.
|
||||
*/
|
||||
#include "libhsakmt.h"
|
||||
|
||||
/* Helper functions for calling KFD SVM ioctl */
|
||||
|
||||
HSAKMT_STATUS HSAKMTAPI hsaKmtSVMSetAttr(void *start_addr, HSAuint64 size,
|
||||
unsigned int nattr,
|
||||
HSA_SVM_ATTRIBUTE *attrs) {
|
||||
CHECK_DXG_OPEN();
|
||||
return HSAKMT_STATUS_NOT_SUPPORTED;
|
||||
}
|
||||
|
||||
HSAKMT_STATUS HSAKMTAPI hsaKmtSVMGetAttr(void *start_addr, HSAuint64 size,
|
||||
unsigned int nattr,
|
||||
HSA_SVM_ATTRIBUTE *attrs) {
|
||||
CHECK_DXG_OPEN();
|
||||
return HSAKMT_STATUS_NOT_SUPPORTED;
|
||||
}
|
||||
|
||||
HSAKMT_STATUS HSAKMTAPI hsaKmtSetXNACKMode(HSAint32 enable) {
|
||||
CHECK_DXG_OPEN();
|
||||
return HSAKMT_STATUS_NOT_SUPPORTED;
|
||||
}
|
||||
|
||||
HSAKMT_STATUS HSAKMTAPI hsaKmtGetXNACKMode(HSAint32 *enable) {
|
||||
CHECK_DXG_OPEN();
|
||||
*enable = false;
|
||||
return HSAKMT_STATUS_SUCCESS;
|
||||
}
|
||||
@@ -0,0 +1,52 @@
|
||||
/*
|
||||
* Copyright © 2014 Advanced Micro Devices, Inc.
|
||||
*
|
||||
* Permission is hereby granted, free of charge, to any person
|
||||
* obtaining a copy of this software and associated documentation
|
||||
* files (the "Software"), to deal in the Software without
|
||||
* restriction, including without limitation the rights to use, copy,
|
||||
* modify, merge, publish, distribute, sublicense, and/or sell copies
|
||||
* of the Software, and to permit persons to whom the Software is
|
||||
* furnished to do so, subject to the following conditions:
|
||||
*
|
||||
* The above copyright notice and this permission notice (including
|
||||
* the next paragraph) shall be included in all copies or substantial
|
||||
* portions of the Software.
|
||||
*
|
||||
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
||||
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
||||
* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
||||
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
|
||||
* HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
|
||||
* WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
||||
* DEALINGS IN THE SOFTWARE.
|
||||
*/
|
||||
#include <iostream>
|
||||
#include <ctime>
|
||||
#include <cstring>
|
||||
#include <cassert>
|
||||
#include "libhsakmt.h"
|
||||
#include "inc/wddm/device.h"
|
||||
|
||||
HSAKMT_STATUS HSAKMTAPI hsaKmtGetClockCounters(HSAuint32 NodeId,
|
||||
HsaClockCounters *Counters) {
|
||||
HSAKMT_STATUS result = HSAKMT_STATUS_SUCCESS;
|
||||
|
||||
CHECK_DXG_OPEN();
|
||||
|
||||
std::memset(Counters, 0, sizeof(*Counters));
|
||||
|
||||
rocr::core::WDDMDevice *device_ = get_wddmdev(NodeId);
|
||||
assert(device_);
|
||||
device_->GetClockCounters(&Counters->GPUClockCounter, nullptr);
|
||||
|
||||
struct timespec ts;
|
||||
if (clock_gettime(CLOCK_REALTIME, &ts) == 0)
|
||||
Counters->CPUClockCounter = ts.tv_sec * 1e9 + ts.tv_nsec;
|
||||
if (clock_gettime(CLOCK_BOOTTIME, &ts) == 0)
|
||||
Counters->SystemClockCounter = ts.tv_sec * 1e9 + ts.tv_nsec;
|
||||
Counters->SystemClockFrequencyHz = 1000000000;
|
||||
|
||||
return result;
|
||||
}
|
||||
+1698
Failā izmaiņas netiks attēlotas, jo tās ir par lielu
Ielādēt izmaiņas
@@ -0,0 +1,519 @@
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
//
|
||||
// The University of Illinois/NCSA
|
||||
// Open Source License (NCSA)
|
||||
//
|
||||
// Copyright (c) 2014-2020, Advanced Micro Devices, Inc. All rights reserved.
|
||||
//
|
||||
// Developed by:
|
||||
//
|
||||
// AMD Research and AMD HSA Software Development
|
||||
//
|
||||
// Advanced Micro Devices, Inc.
|
||||
//
|
||||
// www.amd.com
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
// of this software and associated documentation files (the "Software"), to
|
||||
// deal with the Software without restriction, including without limitation
|
||||
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
// and/or sell copies of the Software, and to permit persons to whom the
|
||||
// Software is furnished to do so, subject to the following conditions:
|
||||
//
|
||||
// - Redistributions of source code must retain the above copyright notice,
|
||||
// this list of conditions and the following disclaimers.
|
||||
// - Redistributions in binary form must reproduce the above copyright
|
||||
// notice, this list of conditions and the following disclaimers in
|
||||
// the documentation and/or other materials provided with the distribution.
|
||||
// - Neither the names of Advanced Micro Devices, Inc,
|
||||
// nor the names of its contributors may be used to endorse or promote
|
||||
// products derived from this Software without specific prior written
|
||||
// permission.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
|
||||
// THE CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
|
||||
// OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
|
||||
// ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
||||
// DEALINGS WITH THE SOFTWARE.
|
||||
//
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
/*
|
||||
Helpers to use native types with C++11 atomic operations.
|
||||
Fixes GCC builtin functionality for x86 with respect to WC and non-temporal
|
||||
stores.
|
||||
*/
|
||||
#ifndef HSA_RUNTIME_CORE_UTIL_ATOMIC_HELPERS_H_
|
||||
#define HSA_RUNTIME_CORE_UTIL_ATOMIC_HELPERS_H_
|
||||
|
||||
#include <atomic>
|
||||
#include "utils.h"
|
||||
|
||||
//ALWAYS_CONSERVATIVE will very likely overfence your code.
|
||||
//For use as a debugging aid only.
|
||||
#define ALWAYS_CONSERVATIVE 0
|
||||
|
||||
#if !ALWAYS_CONSERVATIVE
|
||||
#if defined(__x86_64__) || defined(_M_X64)
|
||||
#define X64_ORDER_WC 1
|
||||
#endif
|
||||
#if X64_ORDER_WC
|
||||
#include <xmmintrin.h>
|
||||
#endif
|
||||
#endif
|
||||
|
||||
namespace rocr {
|
||||
namespace atomic {
|
||||
|
||||
static constexpr int c11ToBuiltInFlags(std::memory_order order)
|
||||
{
|
||||
#if ALWAYS_CONSERVATIVE
|
||||
return __ATOMIC_RELAXED;
|
||||
#elif X64_ORDER_WC
|
||||
return __ATOMIC_RELAXED;
|
||||
#else
|
||||
return (order == std::memory_order_relaxed) ? __ATOMIC_RELAXED :
|
||||
(order == std::memory_order_acquire) ? __ATOMIC_ACQUIRE :
|
||||
(order == std::memory_order_release) ? __ATOMIC_RELEASE :
|
||||
(order == std::memory_order_seq_cst) ? __ATOMIC_SEQ_CST :
|
||||
(order == std::memory_order_consume) ? __ATOMIC_CONSUME :
|
||||
(order == std::memory_order_acq_rel) ? __ATOMIC_ACQ_REL :
|
||||
__ATOMIC_SEQ_CST;
|
||||
#endif
|
||||
}
|
||||
|
||||
static __forceinline void PreFence(std::memory_order order) {
|
||||
#if ALWAYS_CONSERVATIVE
|
||||
switch (order) {
|
||||
case std::memory_order_release:
|
||||
case std::memory_order_seq_cst:
|
||||
case std::memory_order_acq_rel:
|
||||
__atomic_thread_fence(__ATOMIC_SEQ_CST);
|
||||
default:;
|
||||
}
|
||||
#elif X64_ORDER_WC
|
||||
switch (order) {
|
||||
case std::memory_order_release:
|
||||
case std::memory_order_seq_cst:
|
||||
case std::memory_order_acq_rel:
|
||||
_mm_sfence();
|
||||
default:;
|
||||
}
|
||||
#endif
|
||||
}
|
||||
|
||||
static __forceinline void PostFence(std::memory_order order) {
|
||||
#if ALWAYS_CONSERVATIVE
|
||||
switch (order) {
|
||||
case std::memory_order_seq_cst:
|
||||
case std::memory_order_acq_rel:
|
||||
case std::memory_order_acquire:
|
||||
__atomic_thread_fence(__ATOMIC_SEQ_CST);
|
||||
default:;
|
||||
}
|
||||
#elif X64_ORDER_WC
|
||||
switch (order) {
|
||||
case std::memory_order_seq_cst:
|
||||
return _mm_mfence();
|
||||
case std::memory_order_acq_rel:
|
||||
case std::memory_order_acquire:
|
||||
return _mm_lfence();
|
||||
default:;
|
||||
}
|
||||
#endif
|
||||
}
|
||||
|
||||
static __forceinline void Fence(std::memory_order order=std::memory_order_seq_cst) {
|
||||
#if ALWAYS_CONSERVATIVE
|
||||
__atomic_thread_fence(__ATOMIC_SEQ_CST);
|
||||
#elif X64_ORDER_WC
|
||||
switch (order) {
|
||||
case std::memory_order_seq_cst:
|
||||
case std::memory_order_acq_rel:
|
||||
return _mm_mfence();
|
||||
case std::memory_order_acquire:
|
||||
return _mm_lfence();
|
||||
case std::memory_order_release:
|
||||
return _mm_sfence();
|
||||
default:;
|
||||
}
|
||||
#else
|
||||
std::atomic_thread_fence(order);
|
||||
#endif
|
||||
}
|
||||
|
||||
template <class T>
|
||||
static __forceinline void BasicCheck(const T* ptr) {
|
||||
constexpr bool value = __atomic_always_lock_free(sizeof(T), 0);
|
||||
static_assert(value, "Atomic type may not be compatible with peripheral atomics.");
|
||||
};
|
||||
|
||||
template <class T>
|
||||
static __forceinline void BasicCheck(const volatile T* ptr) {
|
||||
constexpr bool value = __atomic_always_lock_free(sizeof(T), 0);
|
||||
static_assert(value, "Atomic type may not be compatible with peripheral atomics.");
|
||||
};
|
||||
|
||||
/// @brief: Load value of type T atomically with specified memory order.
|
||||
/// @param: ptr(Input), a pointer to type T.
|
||||
/// @param: order(Input), memory order with atomic load, relaxed by default.
|
||||
/// @return: T, loaded value.
|
||||
template <class T>
|
||||
static __forceinline T
|
||||
Load(const T* ptr, std::memory_order order = std::memory_order_relaxed) {
|
||||
BasicCheck<T>(ptr);
|
||||
T ret;
|
||||
PreFence(order);
|
||||
__atomic_load(ptr, &ret, c11ToBuiltInFlags(order));
|
||||
PostFence(order);
|
||||
return ret;
|
||||
}
|
||||
|
||||
/// @brief: function overloading, for more info, see previous one.
|
||||
/// @param: ptr(Input), a pointer to volatile type T.
|
||||
/// @param: order(Input), memory order with atomic load, relaxed by default.
|
||||
/// @return: T, loaded value.
|
||||
template <class T>
|
||||
static __forceinline T
|
||||
Load(const volatile T* ptr,
|
||||
std::memory_order order = std::memory_order_relaxed) {
|
||||
BasicCheck<T>(ptr);
|
||||
T ret;
|
||||
PreFence(order);
|
||||
__atomic_load(ptr, &ret, c11ToBuiltInFlags(order));
|
||||
PostFence(order);
|
||||
return ret;
|
||||
}
|
||||
|
||||
/// @brief: Store value of type T with specified memory order.
|
||||
/// @param: ptr(Input), a pointer to instance which will be stored.
|
||||
/// @param: val(Input), value to be stored.
|
||||
/// @param: order(Input), memory order with atomic store, relaxed by default.
|
||||
/// @return: void.
|
||||
template <class T>
|
||||
static __forceinline void Store(
|
||||
T* ptr, T val, std::memory_order order = std::memory_order_relaxed) {
|
||||
BasicCheck<T>(ptr);
|
||||
PreFence(order);
|
||||
__atomic_store(ptr, &val, c11ToBuiltInFlags(order));
|
||||
PostFence(order);
|
||||
}
|
||||
|
||||
/// @brief: Function overloading, for more info, see previous one.
|
||||
/// @param: ptr(Input), a pointer to volatile instance which will be stored.
|
||||
/// @param: val(Input), value to be stored.
|
||||
/// @param: order(Input), memory order with atomic store, relaxed by default.
|
||||
/// @return: void.
|
||||
template <class T>
|
||||
static __forceinline void Store(
|
||||
volatile T* ptr, T val,
|
||||
std::memory_order order = std::memory_order_relaxed) {
|
||||
BasicCheck<T>(ptr);
|
||||
PreFence(order);
|
||||
__atomic_store(ptr, &val, c11ToBuiltInFlags(order));
|
||||
PostFence(order);
|
||||
}
|
||||
|
||||
/// @brief: Compare and swap value atomically with specified memory order.
|
||||
/// @param: ptr(Input), a pointer to variable which is operated on.
|
||||
/// @param: val(Input), value to be stored if condition is satisfied.
|
||||
/// @param: expected(Input), value which is expected.
|
||||
/// @param: order(Input), memory order with atomic operation.
|
||||
/// @return: T, observed value of type T.
|
||||
template <class T>
|
||||
static __forceinline T
|
||||
Cas(T* ptr, T val, T expected,
|
||||
std::memory_order order = std::memory_order_relaxed) {
|
||||
BasicCheck<T>(ptr);
|
||||
PreFence(order);
|
||||
__atomic_compare_exchange(ptr, &expected, &val, false, c11ToBuiltInFlags(order), __ATOMIC_RELAXED);
|
||||
PostFence(order);
|
||||
return expected;
|
||||
}
|
||||
|
||||
/// @brief: Function overloading, for more info, see previous one.
|
||||
/// @param: ptr(Input), a pointer to volatile variable which is operated on.
|
||||
/// @param: val(Input), value to be stored if condition is satisfied.
|
||||
/// @param: expected(Input), value which is expected.
|
||||
/// @param: order(Input), memory order which is relaxed by default.
|
||||
/// @return: T, observed value of type T.
|
||||
template <class T>
|
||||
static __forceinline T
|
||||
Cas(volatile T* ptr, T val, T expected,
|
||||
std::memory_order order = std::memory_order_relaxed) {
|
||||
BasicCheck<T>(ptr);
|
||||
PreFence(order);
|
||||
__atomic_compare_exchange(ptr, &expected, &val, false, c11ToBuiltInFlags(order), __ATOMIC_RELAXED);
|
||||
PostFence(order);
|
||||
return expected;
|
||||
}
|
||||
|
||||
/// @brief: Exchange the value atomically with specified memory order.
|
||||
/// @param: ptr(Input), a pointer to variable which is operated on.
|
||||
/// @param: val(Input), value to be stored.
|
||||
/// @param: order(Input), memory order which is relaxed by default.
|
||||
/// @return: T, the value prior to the exchange.
|
||||
template <class T>
|
||||
static __forceinline T
|
||||
Exchange(T* ptr, T val,
|
||||
std::memory_order order = std::memory_order_relaxed) {
|
||||
BasicCheck<T>(ptr);
|
||||
T ret;
|
||||
PreFence(order);
|
||||
__atomic_exchange(ptr, &val, &ret, c11ToBuiltInFlags(order));
|
||||
PostFence(order);
|
||||
return ret;
|
||||
}
|
||||
|
||||
/// @brief: Function overloading, for more info, see previous one.
|
||||
/// @param: ptr(Input), a pointer to variable which is operated on.
|
||||
/// @param: val(Input), value to be stored.
|
||||
/// @param: order(Input), memory order which is relaxed by default.
|
||||
/// @return: T, the value prior to the exchange.
|
||||
template <class T>
|
||||
static __forceinline T
|
||||
Exchange(volatile T* ptr, T val,
|
||||
std::memory_order order = std::memory_order_relaxed) {
|
||||
BasicCheck<T>(ptr);
|
||||
T ret;
|
||||
PreFence(order);
|
||||
__atomic_exchange(ptr, &val, &ret, c11ToBuiltInFlags(order));
|
||||
PostFence(order);
|
||||
return ret;
|
||||
}
|
||||
|
||||
/// @brief: Add value to variable atomically with specified memory order.
|
||||
/// @param: ptr(Input), a pointer to variable which is operated on.
|
||||
/// @param: val(Input), value to be added.
|
||||
/// @param: order(Input), memory order which is relaxed by default.
|
||||
/// @return: T, the value of the variable prior to the addition.
|
||||
template <class T>
|
||||
static __forceinline T
|
||||
Add(T* ptr, T val, std::memory_order order = std::memory_order_relaxed) {
|
||||
BasicCheck<T>(ptr);
|
||||
PreFence(order);
|
||||
T ret = __atomic_fetch_add(ptr, val, c11ToBuiltInFlags(order));
|
||||
PostFence(order);
|
||||
return ret;
|
||||
}
|
||||
|
||||
/// @brief: Subtract value from the variable atomically with specified memory
|
||||
/// order.
|
||||
/// @param: ptr(Input), a pointer to variable which is operated on.
|
||||
/// @param: val(Input), value to be subtraced.
|
||||
/// @param: order(Input), memory order which is relaxed by default.
|
||||
/// @return: T, value of the variable prior to the subtraction.
|
||||
template <class T>
|
||||
static __forceinline T
|
||||
Sub(T* ptr, T val, std::memory_order order = std::memory_order_relaxed) {
|
||||
BasicCheck<T>(ptr);
|
||||
PreFence(order);
|
||||
T ret = __atomic_fetch_sub(ptr, val, c11ToBuiltInFlags(order));
|
||||
PostFence(order);
|
||||
return ret;
|
||||
}
|
||||
|
||||
/// @brief: Bit And operation on variable atomically with specified memory
|
||||
/// order.
|
||||
/// @param: ptr(Input), a pointer to variable which is operated on.
|
||||
/// @param: val(Input), value which is ANDed with variable.
|
||||
/// @param: order(Input), memory order which is relaxed by default.
|
||||
/// @return: T, value of variable prior to the operation.
|
||||
template <class T>
|
||||
static __forceinline T
|
||||
And(T* ptr, T val, std::memory_order order = std::memory_order_relaxed) {
|
||||
BasicCheck<T>(ptr);
|
||||
PreFence(order);
|
||||
T ret = __atomic_fetch_and(ptr, val, c11ToBuiltInFlags(order));
|
||||
PostFence(order);
|
||||
return ret;
|
||||
}
|
||||
|
||||
/// @brief: Bit Or operation on variable atomically with specified memory order.
|
||||
/// @param: ptr(Input), a pointer to variable which is operated on.
|
||||
/// @param: val(Input), value which is ORed with variable.
|
||||
/// @param: order(Input), memory order which is relaxed by default.
|
||||
/// @return: T, value of variable prior to the operation.
|
||||
template <class T>
|
||||
static __forceinline T
|
||||
Or(T* ptr, T val, std::memory_order order = std::memory_order_relaxed) {
|
||||
BasicCheck<T>(ptr);
|
||||
PreFence(order);
|
||||
T ret = __atomic_fetch_or(ptr, val, c11ToBuiltInFlags(order));
|
||||
PostFence(order);
|
||||
return ret;
|
||||
}
|
||||
|
||||
/// @brief: Bit Xor operation on variable atomically with specified memory
|
||||
/// order.
|
||||
/// @param: ptr(Input), a pointer to variable which is operated on.
|
||||
/// @param: val(Input), value which is XORed with variable.
|
||||
/// @order: order(Input), memory order which is relaxed by default.
|
||||
/// @return: T, valud of variable prior to the opertaion.
|
||||
template <class T>
|
||||
static __forceinline T
|
||||
Xor(T* ptr, T val, std::memory_order order = std::memory_order_relaxed) {
|
||||
BasicCheck<T>(ptr);
|
||||
PreFence(order);
|
||||
T ret = __atomic_fetch_xor(ptr, val, c11ToBuiltInFlags(order));
|
||||
PostFence(order);
|
||||
return ret;
|
||||
}
|
||||
|
||||
/// @brief: Increase the value of variable atomically with specified memory
|
||||
/// order.
|
||||
/// @param: ptr(Input), a pointer to variable which is operated on.
|
||||
/// @param: order(Input), memory order which is relaxed by default.
|
||||
/// @return: T, value of variable prior to the operation.
|
||||
template <class T>
|
||||
static __forceinline T
|
||||
Increment(T* ptr, std::memory_order order = std::memory_order_relaxed) {
|
||||
BasicCheck<T>(ptr);
|
||||
PreFence(order);
|
||||
T ret = __atomic_fetch_add(ptr, 1, c11ToBuiltInFlags(order));
|
||||
PostFence(order);
|
||||
return ret;
|
||||
}
|
||||
|
||||
/// @brief: Decrease the value of the variable atomically with specified memory
|
||||
/// order.
|
||||
/// @param: ptr(Input), a pointer to variable which is operated on.
|
||||
/// @param: order(Input), memory order which is relaxed by default.
|
||||
/// @return: T, value of variable prior to the operation.
|
||||
template <class T>
|
||||
static __forceinline T
|
||||
Decrement(T* ptr, std::memory_order order = std::memory_order_relaxed) {
|
||||
BasicCheck<T>(ptr);
|
||||
PreFence(order);
|
||||
T ret = __atomic_fetch_sub(ptr, 1, c11ToBuiltInFlags(order));
|
||||
PostFence(order);
|
||||
return ret;
|
||||
}
|
||||
|
||||
/// @brief: Add value to variable atomically with specified memory order.
|
||||
/// @param: ptr(Input), a pointer to volatile variable which is operated on.
|
||||
/// @param: val(Input), value to be added.
|
||||
/// @param: order(Input), memory order which is relaxed by default.
|
||||
/// @return: T, the value of the variable prior to the addition.
|
||||
template <class T>
|
||||
static __forceinline T
|
||||
Add(volatile T* ptr, T val,
|
||||
std::memory_order order = std::memory_order_relaxed) {
|
||||
BasicCheck<T>(ptr);
|
||||
PreFence(order);
|
||||
T ret = __atomic_fetch_add(ptr, val, c11ToBuiltInFlags(order));
|
||||
PostFence(order);
|
||||
return ret;
|
||||
}
|
||||
|
||||
/// @brief: Subtract value from the variable atomically with specified memory
|
||||
/// order.
|
||||
/// @param: ptr(Input), a pointer to volatile variable which is operated on.
|
||||
/// @param: val(Input), value to be subtraced.
|
||||
/// @param: order(Input), memory order which is relaxed by default.
|
||||
/// @return: T, value of the variable prior to the subtraction.
|
||||
template <class T>
|
||||
static __forceinline T
|
||||
Sub(volatile T* ptr, T val,
|
||||
std::memory_order order = std::memory_order_relaxed) {
|
||||
BasicCheck<T>(ptr);
|
||||
PreFence(order);
|
||||
T ret = __atomic_fetch_sub(ptr, val, c11ToBuiltInFlags(order));
|
||||
PostFence(order);
|
||||
return ret;
|
||||
}
|
||||
|
||||
/// @brief: Bit And operation on variable atomically with specified memory
|
||||
/// order.
|
||||
/// @param: ptr(Input), a pointer to volatile variable which is operated on.
|
||||
/// @param: val(Input), value which is ANDed with variable.
|
||||
/// @param: order(Input), memory order which is relaxed by default.
|
||||
/// @return: T, value of variable prior to the operation.
|
||||
template <class T>
|
||||
static __forceinline T
|
||||
And(volatile T* ptr, T val,
|
||||
std::memory_order order = std::memory_order_relaxed) {
|
||||
BasicCheck<T>(ptr);
|
||||
PreFence(order);
|
||||
T ret = __atomic_fetch_and(ptr, val, c11ToBuiltInFlags(order));
|
||||
PostFence(order);
|
||||
return ret;
|
||||
}
|
||||
|
||||
/// @brief: Bit Or operation on variable atomically with specified memory order.
|
||||
/// @param: ptr(Input), a pointer to volatile variable which is operated on.
|
||||
/// @param: val(Input), value which is ORed with variable.
|
||||
/// @param: order(Input), memory order which is relaxed by default.
|
||||
/// @return: T, value of variable prior to the operation.
|
||||
template <class T>
|
||||
static __forceinline T Or(volatile T* ptr, T val,
|
||||
std::memory_order order = std::memory_order_relaxed) {
|
||||
BasicCheck<T>(ptr);
|
||||
PreFence(order);
|
||||
T ret = __atomic_fetch_or(ptr, val, c11ToBuiltInFlags(order));
|
||||
PostFence(order);
|
||||
return ret;
|
||||
}
|
||||
|
||||
/// @brief: Bit Xor operation on variable atomically with specified memory
|
||||
/// order.
|
||||
/// @param: ptr(Input), a pointer to volatile variable which is operated on.
|
||||
/// @param: val(Input), value which is XORed with variable.
|
||||
/// @order: order(Input), memory order which is relaxed by default.
|
||||
/// @return: T, valud of variable prior to the opertaion.
|
||||
template <class T>
|
||||
static __forceinline T
|
||||
Xor(volatile T* ptr, T val,
|
||||
std::memory_order order = std::memory_order_relaxed) {
|
||||
BasicCheck<T>(ptr);
|
||||
PreFence(order);
|
||||
T ret = __atomic_fetch_xor(ptr, val, c11ToBuiltInFlags(order));
|
||||
PostFence(order);
|
||||
return ret;
|
||||
}
|
||||
|
||||
/// @brief: Increase the value of variable atomically with specified memory
|
||||
/// order.
|
||||
/// @param: ptr(Input), a pointer to volatile variable which is operated on.
|
||||
/// @param: order(Input), memory order which is relaxed by default.
|
||||
/// @return: T, value of variable prior to the operation.
|
||||
template <class T>
|
||||
static __forceinline T
|
||||
Increment(volatile T* ptr,
|
||||
std::memory_order order = std::memory_order_relaxed) {
|
||||
BasicCheck<T>(ptr);
|
||||
PreFence(order);
|
||||
T ret = __atomic_fetch_add(ptr, 1, c11ToBuiltInFlags(order));
|
||||
PostFence(order);
|
||||
return ret;
|
||||
}
|
||||
|
||||
/// @brief: Decrease the value of the variable atomically with specified memory
|
||||
/// order.
|
||||
/// @param: ptr(Input), a pointer to volatile variable which is operated on.
|
||||
/// @param: order(Input), memory order which is relaxed by default.
|
||||
/// @return: T, value of variable prior to the operation.
|
||||
template <class T>
|
||||
static __forceinline T
|
||||
Decrement(volatile T* ptr,
|
||||
std::memory_order order = std::memory_order_relaxed) {
|
||||
BasicCheck<T>(ptr);
|
||||
PreFence(order);
|
||||
T ret = __atomic_fetch_sub(ptr, 1, c11ToBuiltInFlags(order));
|
||||
PostFence(order);
|
||||
return ret;
|
||||
}
|
||||
} // namespace atomic
|
||||
} // namespace rocr
|
||||
|
||||
#ifdef X64_ORDER_WC
|
||||
#undef X64_ORDER_WC
|
||||
#endif
|
||||
|
||||
#ifdef ALWAYS_CONSERVATIVE
|
||||
#undef ALWAYS_CONSERVATIVE
|
||||
#endif
|
||||
|
||||
#endif // HSA_RUNTIME_CORE_UTIL_ATOMIC_HELPERS_H_
|
||||
@@ -0,0 +1,226 @@
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
//
|
||||
// The University of Illinois/NCSA
|
||||
// Open Source License (NCSA)
|
||||
//
|
||||
// Copyright (c) 2021-2024, Advanced Micro Devices, Inc. All rights reserved.
|
||||
//
|
||||
// Developed by:
|
||||
//
|
||||
// AMD Research and AMD HSA Software Development
|
||||
//
|
||||
// Advanced Micro Devices, Inc.
|
||||
//
|
||||
// www.amd.com
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
// of this software and associated documentation files (the "Software"), to
|
||||
// deal with the Software without restriction, including without limitation
|
||||
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
// and/or sell copies of the Software, and to permit persons to whom the
|
||||
// Software is furnished to do so, subject to the following conditions:
|
||||
//
|
||||
// - Redistributions of source code must retain the above copyright notice,
|
||||
// this list of conditions and the following disclaimers.
|
||||
// - Redistributions in binary form must reproduce the above copyright
|
||||
// notice, this list of conditions and the following disclaimers in
|
||||
// the documentation and/or other materials provided with the distribution.
|
||||
// - Neither the names of Advanced Micro Devices, Inc,
|
||||
// nor the names of its contributors may be used to endorse or promote
|
||||
// products derived from this Software without specific prior written
|
||||
// permission.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIESd OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
|
||||
// THE CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
|
||||
// OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
|
||||
// ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
||||
// DEALINGS WITH THE SOFTWARE.
|
||||
//
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
#include "core/util/flag.h"
|
||||
#include "core/util/utils.h"
|
||||
#include "core/util/os.h"
|
||||
|
||||
#include <vector>
|
||||
#include <map>
|
||||
#include <string>
|
||||
#include <algorithm>
|
||||
#include <locale>
|
||||
|
||||
namespace rocr {
|
||||
FILE* log_file = stderr;
|
||||
uint8_t log_flags[8];
|
||||
|
||||
void log_printf(const char* file, int line, const char* format, ...) {
|
||||
va_list ap;
|
||||
std::stringstream str_thrd_id;
|
||||
str_thrd_id << std::hex << std::this_thread::get_id();
|
||||
va_start(ap, format);
|
||||
char message[4096];
|
||||
vsnprintf(message, sizeof(message), format, ap);
|
||||
va_end(ap);
|
||||
fprintf(log_file, ":%-25s:%-4d: %010lld us: [pid:%-5d tid:0x%s] [***rocr***] %s\n",
|
||||
file, line, os::ReadAccurateClock()/1000ULL, os::GetProcessId(),
|
||||
str_thrd_id.str().c_str(), message);
|
||||
fflush(log_file);
|
||||
}
|
||||
|
||||
// split at separators
|
||||
static std::vector<std::string> split(std::string& str, char sep) {
|
||||
std::vector<std::string> ret;
|
||||
while (!str.empty()) {
|
||||
size_t pos = str.find(sep);
|
||||
if (pos == std::string::npos) {
|
||||
ret.push_back(str);
|
||||
return ret;
|
||||
}
|
||||
ret.push_back(str.substr(0, pos));
|
||||
str.erase(0, pos + 1);
|
||||
}
|
||||
return ret;
|
||||
};
|
||||
|
||||
// Parse id,id-id,... strings into id lists
|
||||
static std::vector<uint32_t> get_elements(std::string& str, uint32_t maxElement) {
|
||||
std::vector<uint32_t> ret;
|
||||
MAKE_NAMED_SCOPE_GUARD(error, [&]() { ret.clear(); });
|
||||
|
||||
std::vector<std::string> ranges = split(str, ',');
|
||||
for (auto& str : ranges) {
|
||||
auto range = split(str, '-');
|
||||
// failure, too many -'s.
|
||||
if (range.size() > 2) return ret;
|
||||
|
||||
char* end;
|
||||
uint32_t index = strtoul(range[0].c_str(), &end, 10);
|
||||
// Invalid syntax - id's must be base 10 digits only.
|
||||
if (*end != '\0') return ret;
|
||||
if (index <= maxElement) ret.push_back(index);
|
||||
|
||||
if (range.size() == 2) {
|
||||
uint32_t secondindex = strtoul(range[1].c_str(), &end, 10);
|
||||
if (*end != '\0') return ret; // bad syntax
|
||||
if (secondindex < index) return ret; // inverted range
|
||||
secondindex = Min(secondindex, maxElement);
|
||||
for (uint32_t i = index + 1; i < secondindex + 1; i++) ret.push_back(i);
|
||||
}
|
||||
}
|
||||
|
||||
// Confirm no duplicate ids.
|
||||
std::sort(ret.begin(), ret.end());
|
||||
if (std::adjacent_find(ret.begin(), ret.end()) != ret.end()) return ret;
|
||||
|
||||
// Good parse, keep result.
|
||||
error.Dismiss();
|
||||
return ret;
|
||||
};
|
||||
|
||||
/*
|
||||
Parse env var per the following syntax, all whitespace is ignored:
|
||||
|
||||
ID = [0-9][0-9]* ex. base 10 numbers
|
||||
ID_list = (ID | ID-ID)[, (ID | ID-ID)]* ex. 0,2-4,7
|
||||
GPU_list = ID_list ex. 0,2-4,7
|
||||
CU_list = 0x[0-F]* | ID_list ex. 0x337F OR 0,2-4,7
|
||||
CU_Set = GPU_list : CU_list ex. 0,2-4,7:0-15,32-47 OR 0,2-4,7:0x337F
|
||||
HSA_CU_MASK = CU_Set [; CU_Set]* ex. 0,2-4,7:0-15,32-47; 3-9:0x337F
|
||||
|
||||
GPU indexes are taken post ROCR_VISIBLE_DEVICES reordering.
|
||||
Listed or bit set CUs will be enabled at queue creation on the associated GPU.
|
||||
All other CUs on the associated GPUs will be disabled.
|
||||
CU masks of unlisted GPUs are not restricted.
|
||||
|
||||
Repeating a GPU or CU ID is a syntax error.
|
||||
Parsing stops at the first CU_Set that has a syntax error, that set and all
|
||||
following sets are ignored.
|
||||
Specifying a mask with no usable CUs (CU_list is 0x0) is a syntax error.
|
||||
Users should use ROCR_VISIBLE_DEVICES if they want to exclude use of a
|
||||
particular GPU.
|
||||
*/
|
||||
void Flag::parse_masks(std::string& var, uint32_t maxGpu, uint32_t maxCU) {
|
||||
if (var.empty()) return;
|
||||
|
||||
// Remove whitespace
|
||||
auto end = std::remove_if(var.begin(), var.end(),
|
||||
[](char c) { return std::isspace<char>(c, std::locale::classic()); });
|
||||
var.erase(end, var.end());
|
||||
|
||||
// Switch to uppercase
|
||||
for (auto& c : var) c = toupper(c);
|
||||
|
||||
// Iterate over cu sets
|
||||
auto sets = split(var, ';');
|
||||
for (auto& set : sets) {
|
||||
auto parts = split(set, ':');
|
||||
if (parts.size() != 2) return;
|
||||
|
||||
// temp storage for cu_set parsing.
|
||||
std::vector<uint32_t> gpu_index;
|
||||
std::vector<uint32_t> mask;
|
||||
|
||||
// parse cu list first, check for bitmask format
|
||||
if (parts[1][1] == 'X') {
|
||||
// Confirm hex format and strip prefix
|
||||
auto& cu = parts[1];
|
||||
if (cu[0] != '0') return;
|
||||
cu.erase(0, 2);
|
||||
|
||||
// Ensure all valid hex characters
|
||||
for (auto& c : cu) {
|
||||
if (!isxdigit(c)) return;
|
||||
}
|
||||
|
||||
// Convert to uint32_t, lsb first.
|
||||
size_t len = cu.length();
|
||||
while (len != 0) {
|
||||
size_t trim = Min(len, size_t(8));
|
||||
len -= trim;
|
||||
auto tmp = cu.substr(len, trim);
|
||||
auto chunk = stoul(tmp, nullptr, 16);
|
||||
mask.push_back(chunk);
|
||||
}
|
||||
|
||||
// Trim dwords beyond maxCUs
|
||||
uint32_t maxDwords = maxCU / 32 + 1;
|
||||
if (maxDwords < mask.size()) mask.resize(maxDwords);
|
||||
|
||||
// Trim leading zeros
|
||||
while (!mask.empty() && mask.back() == 0) mask.pop_back();
|
||||
|
||||
// Mask 0x0 is an error.
|
||||
if (mask.empty()) return;
|
||||
|
||||
} else {
|
||||
// parse cu lists
|
||||
auto cu_indices = get_elements(parts[1], maxCU);
|
||||
if (cu_indices.empty()) return;
|
||||
uint32_t maxdword = cu_indices.back() / 32 + 1;
|
||||
mask.resize(maxdword, 0);
|
||||
for (auto id : cu_indices) {
|
||||
uint32_t index, offset;
|
||||
index = id / 32;
|
||||
offset = id % 32;
|
||||
mask[index] |= 1ul << offset;
|
||||
}
|
||||
}
|
||||
|
||||
// parse device list
|
||||
gpu_index = get_elements(parts[0], maxGpu);
|
||||
if (gpu_index.empty()) return;
|
||||
|
||||
// Ensure that no GPU was repeated across cu_sets
|
||||
for (auto id : gpu_index) {
|
||||
if (cu_mask_.find(id) != cu_mask_.end()) return;
|
||||
}
|
||||
|
||||
// Insert into map
|
||||
for (auto id : gpu_index) {
|
||||
cu_mask_[id] = mask;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace rocr
|
||||
+360
@@ -0,0 +1,360 @@
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
//
|
||||
// The University of Illinois/NCSA
|
||||
// Open Source License (NCSA)
|
||||
//
|
||||
// Copyright (c) 2014-2021, Advanced Micro Devices, Inc. All rights reserved.
|
||||
//
|
||||
// Developed by:
|
||||
//
|
||||
// AMD Research and AMD HSA Software Development
|
||||
//
|
||||
// Advanced Micro Devices, Inc.
|
||||
//
|
||||
// www.amd.com
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
// of this software and associated documentation files (the "Software"), to
|
||||
// deal with the Software without restriction, including without limitation
|
||||
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
// and/or sell copies of the Software, and to permit persons to whom the
|
||||
// Software is furnished to do so, subject to the following conditions:
|
||||
//
|
||||
// - Redistributions of source code must retain the above copyright notice,
|
||||
// this list of conditions and the following disclaimers.
|
||||
// - Redistributions in binary form must reproduce the above copyright
|
||||
// notice, this list of conditions and the following disclaimers in
|
||||
// the documentation and/or other materials provided with the distribution.
|
||||
// - Neither the names of Advanced Micro Devices, Inc,
|
||||
// nor the names of its contributors may be used to endorse or promote
|
||||
// products derived from this Software without specific prior written
|
||||
// permission.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIESd OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
|
||||
// THE CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
|
||||
// OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
|
||||
// ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
||||
// DEALINGS WITH THE SOFTWARE.
|
||||
//
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
#ifndef HSA_RUNTIME_CORE_INC_FLAG_H_
|
||||
#define HSA_RUNTIME_CORE_INC_FLAG_H_
|
||||
|
||||
#include <stdint.h>
|
||||
|
||||
#include <vector>
|
||||
#include <map>
|
||||
#include <string>
|
||||
|
||||
#include "core/util/os.h"
|
||||
#include "core/util/utils.h"
|
||||
|
||||
namespace rocr {
|
||||
|
||||
class Flag {
|
||||
public:
|
||||
enum SDMA_OVERRIDE { SDMA_DISABLE, SDMA_ENABLE, SDMA_DEFAULT };
|
||||
enum SRAMECC_ENABLE { SRAMECC_DISABLED, SRAMECC_ENABLED, SRAMECC_DEFAULT };
|
||||
|
||||
// The values are meaningful and chosen to satisfy the thunk API.
|
||||
enum XNACK_REQUEST { XNACK_DISABLE = 0, XNACK_ENABLE = 1, XNACK_UNCHANGED = 2 };
|
||||
static_assert(XNACK_DISABLE == 0, "XNACK_REQUEST enum values improperly changed.");
|
||||
static_assert(XNACK_ENABLE == 1, "XNACK_REQUEST enum values improperly changed.");
|
||||
|
||||
// Lift limit for 2.10 release RCCL workaround.
|
||||
const size_t DEFAULT_SCRATCH_SINGLE_LIMIT = 146800640; // small_limit >> 2;
|
||||
|
||||
explicit Flag() { Refresh(); }
|
||||
|
||||
virtual ~Flag() {}
|
||||
|
||||
void Refresh() {
|
||||
std::string var = os::GetEnvVar("HSA_CHECK_FLAT_SCRATCH");
|
||||
check_flat_scratch_ = (var == "1") ? true : false;
|
||||
|
||||
var = os::GetEnvVar("HSA_ENABLE_VM_FAULT_MESSAGE");
|
||||
enable_vm_fault_message_ = (var == "0") ? false : true;
|
||||
|
||||
var = os::GetEnvVar("HSA_ENABLE_QUEUE_FAULT_MESSAGE");
|
||||
enable_queue_fault_message_ = (var == "0") ? false : true;
|
||||
|
||||
var = os::GetEnvVar("HSA_ENABLE_INTERRUPT");
|
||||
enable_interrupt_ = (var == "0") ? false : true;
|
||||
|
||||
var = os::GetEnvVar("HSA_ENABLE_SDMA");
|
||||
enable_sdma_ = (var == "0") ? SDMA_DISABLE : ((var == "1") ? SDMA_ENABLE : SDMA_DEFAULT);
|
||||
|
||||
var = os::GetEnvVar("HSA_ENABLE_PEER_SDMA");
|
||||
enable_peer_sdma_ = (var == "0") ? SDMA_DISABLE : ((var == "1") ? SDMA_ENABLE : SDMA_DEFAULT);
|
||||
|
||||
var = os::GetEnvVar("HSA_ENABLE_SDMA_GANG");
|
||||
enable_sdma_gang_ = (var == "0") ? SDMA_DISABLE :
|
||||
((var == "1") ? SDMA_ENABLE : SDMA_DEFAULT);
|
||||
|
||||
var = os::GetEnvVar("HSA_ENABLE_SDMA_COPY_SIZE_OVERRIDE");
|
||||
enable_sdma_copy_size_override_ = (var == "0") ? SDMA_DISABLE :
|
||||
((var == "1") ? SDMA_ENABLE : SDMA_DEFAULT);
|
||||
|
||||
visible_gpus_ = os::GetEnvVar("ROCR_VISIBLE_DEVICES");
|
||||
filter_visible_gpus_ = os::IsEnvVarSet("ROCR_VISIBLE_DEVICES");
|
||||
|
||||
var = os::GetEnvVar("HSA_RUNNING_UNDER_VALGRIND");
|
||||
running_valgrind_ = (var == "1") ? true : false;
|
||||
|
||||
var = os::GetEnvVar("HSA_SDMA_WAIT_IDLE");
|
||||
sdma_wait_idle_ = (var == "1") ? true : false;
|
||||
|
||||
var = os::GetEnvVar("HSA_MAX_QUEUES");
|
||||
max_queues_ = static_cast<uint32_t>(atoi(var.c_str()));
|
||||
|
||||
// Maximum amount of scratch mem that can be used per process per gpu
|
||||
var = os::GetEnvVar("HSA_SCRATCH_MEM");
|
||||
scratch_mem_size_ = atoi(var.c_str());
|
||||
|
||||
// Scratch memory sizes > HSA_SCRATCH_SINGLE_LIMIT will trigger a use-once scheme
|
||||
// We also reserve HSA_SCRATCH_SINGLE_LIMIT per process per gpu to guarrantee we
|
||||
// have sufficient memory to for scratch in case user tried to allocate all device
|
||||
// memory
|
||||
if (os::IsEnvVarSet("HSA_SCRATCH_SINGLE_LIMIT")) {
|
||||
var = os::GetEnvVar("HSA_SCRATCH_SINGLE_LIMIT");
|
||||
scratch_single_limit_ = atoi(var.c_str());
|
||||
} else {
|
||||
scratch_single_limit_ = DEFAULT_SCRATCH_SINGLE_LIMIT;
|
||||
}
|
||||
|
||||
tools_lib_names_ = os::GetEnvVar("HSA_TOOLS_LIB");
|
||||
|
||||
var = os::GetEnvVar("HSA_TOOLS_REPORT_LOAD_FAILURE");
|
||||
|
||||
ifdebug {
|
||||
report_tool_load_failures_ = (var == "1") ? true : false;
|
||||
} else {
|
||||
report_tool_load_failures_ = (var == "0") ? false : true;
|
||||
}
|
||||
|
||||
var = os::GetEnvVar("HSA_DISABLE_FRAGMENT_ALLOCATOR");
|
||||
disable_fragment_alloc_ = (var == "1") ? true : false;
|
||||
|
||||
var = os::GetEnvVar("HSA_ENABLE_SDMA_HDP_FLUSH");
|
||||
enable_sdma_hdp_flush_ = (var == "0") ? false : true;
|
||||
|
||||
var = os::GetEnvVar("HSA_REV_COPY_DIR");
|
||||
rev_copy_dir_ = (var == "1") ? true : false;
|
||||
|
||||
var = os::GetEnvVar("HSA_FORCE_FINE_GRAIN_PCIE");
|
||||
fine_grain_pcie_ = (var == "1") ? true : false;
|
||||
|
||||
var = os::GetEnvVar("HSA_NO_SCRATCH_RECLAIM");
|
||||
no_scratch_reclaim_ = (var == "1") ? true : false;
|
||||
|
||||
var = os::GetEnvVar("HSA_NO_SCRATCH_THREAD_LIMITER");
|
||||
no_scratch_thread_limit_ = (var == "1") ? true : false;
|
||||
|
||||
var = os::GetEnvVar("HSA_DISABLE_IMAGE");
|
||||
disable_image_ = (var == "1") ? true : false;
|
||||
|
||||
var = os::GetEnvVar("HSA_DISABLE_PC_SAMPLING");
|
||||
disable_pc_sampling_ = (var == "1") ? true : false;
|
||||
|
||||
var = os::GetEnvVar("HSA_LOADER_ENABLE_MMAP_URI");
|
||||
loader_enable_mmap_uri_ = (var == "1") ? true : false;
|
||||
|
||||
var = os::GetEnvVar("HSA_FORCE_SDMA_SIZE");
|
||||
force_sdma_size_ = var.empty() ? 1024 * 1024 : atoi(var.c_str());
|
||||
|
||||
var = os::GetEnvVar("HSA_IGNORE_SRAMECC_MISREPORT");
|
||||
check_sramecc_validity_ = (var == "1") ? false : true;
|
||||
|
||||
// Legal values are zero "0" or one "1". Any other value will
|
||||
// be interpreted as not defining the env variable.
|
||||
var = os::GetEnvVar("HSA_XNACK");
|
||||
xnack_ = (var == "0") ? XNACK_DISABLE : ((var == "1") ? XNACK_ENABLE : XNACK_UNCHANGED);
|
||||
|
||||
var = os::GetEnvVar("HSA_ENABLE_DEBUG");
|
||||
debug_ = (var == "1") ? true : false;
|
||||
|
||||
var = os::GetEnvVar("HSA_CU_MASK_SKIP_INIT");
|
||||
cu_mask_skip_init_ = (var == "1") ? true : false;
|
||||
|
||||
// Temporary opt-in for corrected HSA_AMD_AGENT_INFO_COOPERATIVE_COMPUTE_UNIT_COUNT behavior.
|
||||
// Will become opt-out and possibly removed in future releases.
|
||||
var = os::GetEnvVar("HSA_COOP_CU_COUNT");
|
||||
coop_cu_count_ = (var == "1") ? true : false;
|
||||
|
||||
var = os::GetEnvVar("HSA_DISCOVER_COPY_AGENTS");
|
||||
discover_copy_agents_ = (var == "1") ? true : false;
|
||||
|
||||
var = os::GetEnvVar("HSA_SVM_PROFILE");
|
||||
svm_profile_ = var;
|
||||
|
||||
var = os::GetEnvVar("HSA_ENABLE_SRAMECC");
|
||||
sramecc_enable_ =
|
||||
(var == "0") ? SRAMECC_DISABLED : ((var == "1") ? SRAMECC_ENABLED : SRAMECC_DEFAULT);
|
||||
|
||||
var = os::GetEnvVar("HSA_IMAGE_PRINT_SRD");
|
||||
image_print_srd_ = (var == "1") ? true : false;
|
||||
|
||||
var = os::GetEnvVar("HSA_ENABLE_MWAITX");
|
||||
enable_mwaitx_ = (var == "1") ? true : false;
|
||||
|
||||
// Temporary environment variable to disable CPU affinity override
|
||||
// Will either rename to HSA_OVERRIDE_CPU_AFFINITY later or remove completely.
|
||||
var = os::GetEnvVar("HSA_OVERRIDE_CPU_AFFINITY_DEBUG");
|
||||
override_cpu_affinity_ = (var == "0") ? false : true;
|
||||
}
|
||||
|
||||
void parse_masks(uint32_t maxGpu, uint32_t maxCU) {
|
||||
std::string var = os::GetEnvVar("HSA_CU_MASK");
|
||||
parse_masks(var, maxGpu, maxCU);
|
||||
}
|
||||
|
||||
bool check_flat_scratch() const { return check_flat_scratch_; }
|
||||
|
||||
bool enable_vm_fault_message() const { return enable_vm_fault_message_; }
|
||||
|
||||
bool enable_queue_fault_message() const { return enable_queue_fault_message_; }
|
||||
|
||||
bool enable_interrupt() const { return enable_interrupt_; }
|
||||
|
||||
bool enable_sdma_hdp_flush() const { return enable_sdma_hdp_flush_; }
|
||||
|
||||
bool running_valgrind() const { return running_valgrind_; }
|
||||
|
||||
bool sdma_wait_idle() const { return sdma_wait_idle_; }
|
||||
|
||||
bool report_tool_load_failures() const { return report_tool_load_failures_; }
|
||||
|
||||
bool disable_fragment_alloc() const { return disable_fragment_alloc_; }
|
||||
|
||||
bool rev_copy_dir() const { return rev_copy_dir_; }
|
||||
|
||||
bool fine_grain_pcie() const { return fine_grain_pcie_; }
|
||||
|
||||
bool no_scratch_reclaim() const { return no_scratch_reclaim_; }
|
||||
|
||||
bool no_scratch_thread_limiter() const { return no_scratch_thread_limit_; }
|
||||
|
||||
SDMA_OVERRIDE enable_sdma() const { return enable_sdma_; }
|
||||
|
||||
SDMA_OVERRIDE enable_peer_sdma() const { return enable_peer_sdma_; }
|
||||
|
||||
SDMA_OVERRIDE enable_sdma_gang() const { return enable_sdma_gang_; }
|
||||
|
||||
SDMA_OVERRIDE enable_sdma_copy_size_override() const { return enable_sdma_copy_size_override_; }
|
||||
|
||||
std::string visible_gpus() const { return visible_gpus_; }
|
||||
|
||||
bool filter_visible_gpus() const { return filter_visible_gpus_; }
|
||||
|
||||
uint32_t max_queues() const { return max_queues_; }
|
||||
|
||||
size_t scratch_mem_size() const { return scratch_mem_size_; }
|
||||
|
||||
size_t scratch_single_limit() const { return scratch_single_limit_; }
|
||||
|
||||
std::string tools_lib_names() const { return tools_lib_names_; }
|
||||
|
||||
bool disable_image() const { return disable_image_; }
|
||||
|
||||
bool disable_pc_sampling() const { return disable_pc_sampling_; }
|
||||
|
||||
bool loader_enable_mmap_uri() const { return loader_enable_mmap_uri_; }
|
||||
|
||||
size_t force_sdma_size() const { return force_sdma_size_; }
|
||||
|
||||
bool check_sramecc_validity() const { return check_sramecc_validity_; }
|
||||
|
||||
bool override_cpu_affinity() const { return override_cpu_affinity_; }
|
||||
|
||||
bool image_print_srd() const { return image_print_srd_; }
|
||||
|
||||
bool check_mwaitx(bool mwaitx_supported) {
|
||||
if (enable_mwaitx_ && !mwaitx_supported) enable_mwaitx_ = false;
|
||||
|
||||
return enable_mwaitx_;
|
||||
}
|
||||
|
||||
XNACK_REQUEST xnack() const { return xnack_; }
|
||||
|
||||
bool debug() const { return debug_; }
|
||||
|
||||
const std::vector<uint32_t>& cu_mask(uint32_t gpu_index) const {
|
||||
static const std::vector<uint32_t> empty;
|
||||
auto it = cu_mask_.find(gpu_index);
|
||||
if (it == cu_mask_.end()) return empty;
|
||||
return it->second;
|
||||
}
|
||||
|
||||
bool cu_mask_skip_init() const { return cu_mask_skip_init_; }
|
||||
|
||||
bool coop_cu_count() const { return coop_cu_count_; }
|
||||
|
||||
bool discover_copy_agents() const { return discover_copy_agents_; }
|
||||
|
||||
const std::string& svm_profile() const { return svm_profile_; }
|
||||
|
||||
SRAMECC_ENABLE sramecc_enable() const { return sramecc_enable_; }
|
||||
|
||||
private:
|
||||
bool check_flat_scratch_;
|
||||
bool enable_vm_fault_message_;
|
||||
bool enable_interrupt_;
|
||||
bool enable_sdma_hdp_flush_;
|
||||
bool running_valgrind_;
|
||||
bool sdma_wait_idle_;
|
||||
bool enable_queue_fault_message_;
|
||||
bool report_tool_load_failures_;
|
||||
bool disable_fragment_alloc_;
|
||||
bool rev_copy_dir_;
|
||||
bool fine_grain_pcie_;
|
||||
bool no_scratch_reclaim_;
|
||||
bool no_scratch_thread_limit_;
|
||||
bool disable_image_;
|
||||
bool disable_pc_sampling_;
|
||||
bool loader_enable_mmap_uri_;
|
||||
bool check_sramecc_validity_;
|
||||
bool debug_;
|
||||
bool cu_mask_skip_init_;
|
||||
bool coop_cu_count_;
|
||||
bool discover_copy_agents_;
|
||||
bool override_cpu_affinity_;
|
||||
bool image_print_srd_;
|
||||
bool enable_mwaitx_;
|
||||
|
||||
SDMA_OVERRIDE enable_sdma_;
|
||||
SDMA_OVERRIDE enable_peer_sdma_;
|
||||
SDMA_OVERRIDE enable_sdma_gang_;
|
||||
SDMA_OVERRIDE enable_sdma_copy_size_override_;
|
||||
|
||||
bool filter_visible_gpus_;
|
||||
std::string visible_gpus_;
|
||||
|
||||
uint32_t max_queues_;
|
||||
|
||||
size_t scratch_mem_size_;
|
||||
size_t scratch_single_limit_;
|
||||
|
||||
std::string tools_lib_names_;
|
||||
std::string svm_profile_;
|
||||
|
||||
size_t force_sdma_size_;
|
||||
|
||||
// Indicates user preference for Xnack state.
|
||||
XNACK_REQUEST xnack_;
|
||||
|
||||
SRAMECC_ENABLE sramecc_enable_;
|
||||
|
||||
// Map GPU index post RVD to its default cu mask.
|
||||
std::map<uint32_t, std::vector<uint32_t>> cu_mask_;
|
||||
|
||||
void parse_masks(std::string& args, uint32_t maxGpu, uint32_t maxCU);
|
||||
|
||||
DISALLOW_COPY_AND_ASSIGN(Flag);
|
||||
};
|
||||
|
||||
} // namespace rocr
|
||||
|
||||
#endif // header guard
|
||||
@@ -0,0 +1,155 @@
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
//
|
||||
// The University of Illinois/NCSA
|
||||
// Open Source License (NCSA)
|
||||
//
|
||||
// Copyright (c) 2014-2020, Advanced Micro Devices, Inc. All rights reserved.
|
||||
//
|
||||
// Developed by:
|
||||
//
|
||||
// AMD Research and AMD HSA Software Development
|
||||
//
|
||||
// Advanced Micro Devices, Inc.
|
||||
//
|
||||
// www.amd.com
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
// of this software and associated documentation files (the "Software"), to
|
||||
// deal with the Software without restriction, including without limitation
|
||||
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
// and/or sell copies of the Software, and to permit persons to whom the
|
||||
// Software is furnished to do so, subject to the following conditions:
|
||||
//
|
||||
// - Redistributions of source code must retain the above copyright notice,
|
||||
// this list of conditions and the following disclaimers.
|
||||
// - Redistributions in binary form must reproduce the above copyright
|
||||
// notice, this list of conditions and the following disclaimers in
|
||||
// the documentation and/or other materials provided with the distribution.
|
||||
// - Neither the names of Advanced Micro Devices, Inc,
|
||||
// nor the names of its contributors may be used to endorse or promote
|
||||
// products derived from this Software without specific prior written
|
||||
// permission.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIESd OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
|
||||
// THE CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
|
||||
// OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
|
||||
// ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
||||
// DEALINGS WITH THE SOFTWARE.
|
||||
//
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
#ifndef HSA_RUNTIME_CORE_UTIL_LAZY_PTR_H_
|
||||
#define HSA_RUNTIME_CORE_UTIL_LAZY_PTR_H_
|
||||
|
||||
#include <memory>
|
||||
#include <utility>
|
||||
#include <functional>
|
||||
|
||||
#include "core/util/locks.h"
|
||||
#include "core/util/utils.h"
|
||||
|
||||
namespace rocr {
|
||||
|
||||
/*
|
||||
* Wrapper for a std::unique_ptr that initializes its object at first use.
|
||||
*/
|
||||
template <typename T> class lazy_ptr {
|
||||
public:
|
||||
lazy_ptr() {}
|
||||
|
||||
explicit lazy_ptr(std::function<T*()> Constructor) { reset(Constructor); }
|
||||
|
||||
lazy_ptr(lazy_ptr&& rhs) {
|
||||
obj = std::move(rhs.obj);
|
||||
func = std::move(rhs.func);
|
||||
}
|
||||
|
||||
lazy_ptr& operator=(lazy_ptr&& rhs) {
|
||||
obj = std::move(rhs.obj);
|
||||
func = std::move(rhs.func);
|
||||
}
|
||||
|
||||
lazy_ptr(lazy_ptr&) = delete;
|
||||
lazy_ptr& operator=(lazy_ptr&) = delete;
|
||||
|
||||
void reset(std::function<T*()> Constructor = nullptr) {
|
||||
obj.reset();
|
||||
func = Constructor;
|
||||
}
|
||||
|
||||
void reset(T* ptr) {
|
||||
obj.reset(ptr);
|
||||
func = nullptr;
|
||||
}
|
||||
|
||||
bool operator==(T* rhs) const { return obj.get() == rhs; }
|
||||
bool operator!=(T* rhs) const { return obj.get() != rhs; }
|
||||
|
||||
const std::unique_ptr<T>& operator->() const {
|
||||
make(true);
|
||||
assert(obj != nullptr && "Null dereference through lazy_ptr.");
|
||||
return obj;
|
||||
}
|
||||
|
||||
std::unique_ptr<T>& operator*() {
|
||||
make(true);
|
||||
return obj;
|
||||
}
|
||||
|
||||
const std::unique_ptr<T>& operator*() const {
|
||||
make(true);
|
||||
return obj;
|
||||
}
|
||||
|
||||
/*
|
||||
* Ensures that the object is created or is being created.
|
||||
* This is useful when early construction of the object is required.
|
||||
*/
|
||||
void touch() const { make(false); }
|
||||
|
||||
// Tells if the lazy object has been constructed or not.
|
||||
// Construction may fail silently (return nullptr).
|
||||
bool created() const {
|
||||
std::atomic_thread_fence(std::memory_order_acquire);
|
||||
return func == nullptr;
|
||||
}
|
||||
|
||||
// Tells if the lazy object exists or not.
|
||||
bool empty() const {
|
||||
std::atomic_thread_fence(std::memory_order_acquire);
|
||||
return obj == nullptr;
|
||||
}
|
||||
|
||||
private:
|
||||
mutable std::unique_ptr<T> obj;
|
||||
mutable std::function<T*(void)> func;
|
||||
mutable KernelMutex lock;
|
||||
|
||||
// Separated from make to improve inlining.
|
||||
void make_body(bool block) const {
|
||||
if (block) {
|
||||
lock.Acquire();
|
||||
} else if (!lock.Try()) {
|
||||
return;
|
||||
}
|
||||
MAKE_SCOPE_GUARD([&]() { lock.Release(); });
|
||||
if (func == nullptr) return;
|
||||
T* ptr = func();
|
||||
obj.reset(ptr);
|
||||
std::atomic_thread_fence(std::memory_order_release);
|
||||
func = nullptr;
|
||||
}
|
||||
|
||||
__forceinline void make(bool block) const {
|
||||
if (!created()) {
|
||||
make_body(block);
|
||||
}
|
||||
}
|
||||
|
||||
};
|
||||
|
||||
} // namespace rocr
|
||||
|
||||
#endif // HSA_RUNTIME_CORE_UTIL_LAZY_PTR_H_
|
||||
@@ -0,0 +1,771 @@
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
//
|
||||
// The University of Illinois/NCSA
|
||||
// Open Source License (NCSA)
|
||||
//
|
||||
// Copyright (c) 2014-2024, Advanced Micro Devices, Inc. All rights reserved.
|
||||
//
|
||||
// Developed by:
|
||||
//
|
||||
// AMD Research and AMD HSA Software Development
|
||||
//
|
||||
// Advanced Micro Devices, Inc.
|
||||
//
|
||||
// www.amd.com
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
// of this software and associated documentation files (the "Software"), to
|
||||
// deal with the Software without restriction, including without limitation
|
||||
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
// and/or sell copies of the Software, and to permit persons to whom the
|
||||
// Software is furnished to do so, subject to the following conditions:
|
||||
//
|
||||
// - Redistributions of source code must retain the above copyright notice,
|
||||
// this list of conditions and the following disclaimers.
|
||||
// - Redistributions in binary form must reproduce the above copyright
|
||||
// notice, this list of conditions and the following disclaimers in
|
||||
// the documentation and/or other materials provided with the distribution.
|
||||
// - Neither the names of Advanced Micro Devices, Inc,
|
||||
// nor the names of its contributors may be used to endorse or promote
|
||||
// products derived from this Software without specific prior written
|
||||
// permission.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
|
||||
// THE CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
|
||||
// OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
|
||||
// ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
||||
// DEALINGS WITH THE SOFTWARE.
|
||||
//
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
#ifdef __linux__
|
||||
#include "core/util/os.h"
|
||||
#include "core/util/utils.h"
|
||||
|
||||
#include <link.h>
|
||||
#include <dlfcn.h>
|
||||
#include <pthread.h>
|
||||
#include <limits.h>
|
||||
#include <sched.h>
|
||||
#include <sys/sysinfo.h>
|
||||
#include <sys/time.h>
|
||||
#include <sys/utsname.h>
|
||||
#include <unistd.h>
|
||||
#include <errno.h>
|
||||
#include <cstring>
|
||||
#include <atomic>
|
||||
#include <memory>
|
||||
#include <string>
|
||||
#include <utility>
|
||||
#include <semaphore.h>
|
||||
#include "core/inc/runtime.h"
|
||||
#if defined(__i386__) || defined(__x86_64__)
|
||||
#include <cpuid.h>
|
||||
#endif
|
||||
|
||||
namespace rocr {
|
||||
namespace os {
|
||||
|
||||
struct ThreadArgs {
|
||||
void* entry_args;
|
||||
ThreadEntry entry_function;
|
||||
};
|
||||
|
||||
void* __stdcall ThreadTrampoline(void* arg) {
|
||||
ThreadArgs* ar = (ThreadArgs*)arg;
|
||||
ThreadEntry CallMe = ar->entry_function;
|
||||
void* Data = ar->entry_args;
|
||||
delete ar;
|
||||
CallMe(Data);
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
// Thread container allows multiple waits and separate close (destroy).
|
||||
class os_thread {
|
||||
public:
|
||||
explicit os_thread(ThreadEntry function, void* threadArgument, uint stackSize)
|
||||
: thread(0), lock(nullptr), state(RUNNING) {
|
||||
int err;
|
||||
std::unique_ptr<ThreadArgs> args(new ThreadArgs);
|
||||
lock = CreateMutex();
|
||||
if (lock == nullptr) return;
|
||||
|
||||
args->entry_args = threadArgument;
|
||||
args->entry_function = function;
|
||||
|
||||
pthread_attr_t attrib;
|
||||
err = pthread_attr_init(&attrib);
|
||||
if (err != 0) {
|
||||
fprintf(stderr, "pthread_attr_init failed: %s\n", strerror(err));
|
||||
return;
|
||||
}
|
||||
|
||||
if (stackSize != 0) {
|
||||
stackSize = Max(uint(PTHREAD_STACK_MIN), stackSize);
|
||||
stackSize = AlignUp(stackSize, 4096);
|
||||
err = pthread_attr_setstacksize(&attrib, stackSize);
|
||||
if (err != 0) {
|
||||
fprintf(stderr, "pthread_attr_setstacksize failed: %s\n", strerror(err));
|
||||
err = pthread_attr_destroy(&attrib);
|
||||
if (err != 0) {
|
||||
fprintf(stderr, "pthread_attr_destroy failed: %s\n", strerror(err));
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
int cores = 0;
|
||||
cpu_set_t* cpuset = nullptr;
|
||||
|
||||
if (core::Runtime::runtime_singleton_->flag().override_cpu_affinity()) {
|
||||
cores = get_nprocs_conf();
|
||||
cpuset = CPU_ALLOC(cores);
|
||||
if (cpuset == nullptr) {
|
||||
fprintf(stderr, "CPU_ALLOC failed: %s\n", strerror(errno));
|
||||
return;
|
||||
}
|
||||
CPU_ZERO_S(CPU_ALLOC_SIZE(cores), cpuset);
|
||||
for (int i = 0; i < cores; i++) {
|
||||
CPU_SET_S(i, CPU_ALLOC_SIZE(cores), cpuset);
|
||||
}
|
||||
err = pthread_attr_setaffinity_np(&attrib, CPU_ALLOC_SIZE(cores), cpuset);
|
||||
CPU_FREE(cpuset);
|
||||
if (err != 0) {
|
||||
fprintf(stderr, "pthread_setaffinity_np failed: %s\n", strerror(err));
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
err = pthread_create(&thread, &attrib, ThreadTrampoline, args.get());
|
||||
|
||||
// Probably a stack size error since system limits can be different from PTHREAD_STACK_MIN
|
||||
// Attempt to grow the stack within reason.
|
||||
if ((err == EINVAL) && stackSize != 0) {
|
||||
while (stackSize < 20 * 1024 * 1024) {
|
||||
stackSize *= 2;
|
||||
err = pthread_attr_setstacksize(&attrib, stackSize);
|
||||
if (err != 0) {
|
||||
fprintf(stderr, "pthread_attr_setstacksize failed: %s\n", strerror(err));
|
||||
return;
|
||||
}
|
||||
err = pthread_create(&thread, &attrib, ThreadTrampoline, args.get());
|
||||
if (err != EINVAL) break;
|
||||
debug_print("pthread_create returned EINVAL, doubling stack size\n");
|
||||
}
|
||||
}
|
||||
|
||||
if (err == 0)
|
||||
args.release();
|
||||
else
|
||||
thread = 0;
|
||||
|
||||
err = pthread_attr_destroy(&attrib);
|
||||
if (err != 0) {
|
||||
fprintf(stderr, "pthread_attr_destroy failed: %s\n", strerror(err));
|
||||
}
|
||||
}
|
||||
|
||||
os_thread(os_thread&& rhs) {
|
||||
thread = rhs.thread;
|
||||
lock = rhs.lock;
|
||||
state = int(rhs.state);
|
||||
rhs.thread = 0;
|
||||
rhs.lock = nullptr;
|
||||
}
|
||||
|
||||
os_thread(os_thread&) = delete;
|
||||
|
||||
~os_thread() {
|
||||
if (lock != nullptr) DestroyMutex(lock);
|
||||
if ((state == RUNNING) && (thread != 0)) {
|
||||
int err = pthread_detach(thread);
|
||||
if (err != 0) fprintf(stderr, "pthread_detach failed: %s\n", strerror(err));
|
||||
}
|
||||
}
|
||||
|
||||
bool Valid() { return (lock != nullptr) && (thread != 0); }
|
||||
|
||||
bool Wait() {
|
||||
if (state == FINISHED) return true;
|
||||
AcquireMutex(lock);
|
||||
if (state == FINISHED) {
|
||||
ReleaseMutex(lock);
|
||||
return true;
|
||||
}
|
||||
int err = pthread_join(thread, NULL);
|
||||
bool success = (err == 0);
|
||||
if (success) state = FINISHED;
|
||||
ReleaseMutex(lock);
|
||||
return success;
|
||||
}
|
||||
|
||||
private:
|
||||
pthread_t thread;
|
||||
Mutex lock;
|
||||
std::atomic<int> state;
|
||||
enum { FINISHED = 0, RUNNING = 1 };
|
||||
};
|
||||
|
||||
static_assert(sizeof(LibHandle) == sizeof(void*), "OS abstraction size mismatch");
|
||||
static_assert(sizeof(Semaphore) == sizeof(sem_t*), "OS abstraction size mismatch");
|
||||
static_assert(sizeof(Mutex) == sizeof(pthread_mutex_t*), "OS abstraction size mismatch");
|
||||
static_assert(sizeof(SharedMutex) == sizeof(pthread_rwlock_t*), "OS abstraction size mismatch");
|
||||
static_assert(sizeof(Thread) == sizeof(os_thread*), "OS abstraction size mismatch");
|
||||
|
||||
LibHandle LoadLib(std::string filename) {
|
||||
void* ret = dlopen(filename.c_str(), RTLD_LAZY);
|
||||
if (ret == nullptr) debug_print("LoadLib(%s) failed: %s\n", filename.c_str(), dlerror());
|
||||
return *(LibHandle*)&ret;
|
||||
}
|
||||
|
||||
void* GetExportAddress(LibHandle lib, std::string export_name) {
|
||||
void* ret = dlsym(*(void**)&lib, export_name.c_str());
|
||||
|
||||
// dlsym searches the given library and all the library's load dependencies.
|
||||
// Remaining code limits symbol lookup to only the library handle given.
|
||||
// This lookup pattern matches Windows.
|
||||
if (ret == NULL) return ret;
|
||||
|
||||
link_map* map;
|
||||
int err = dlinfo(*(void**)&lib, RTLD_DI_LINKMAP, &map);
|
||||
if (err == -1) {
|
||||
fprintf(stderr, "dlinfo failed: %s\n", dlerror());
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
Dl_info info;
|
||||
err = dladdr(ret, &info);
|
||||
if (err == 0) {
|
||||
fprintf(stderr, "dladdr failed.\n");
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
if (strcmp(info.dli_fname, map->l_name) == 0) return ret;
|
||||
|
||||
return NULL;
|
||||
}
|
||||
|
||||
void CloseLib(LibHandle lib) { dlclose(*(void**)&lib); }
|
||||
|
||||
/*
|
||||
* @brief Look for a symbol called "HSA_AMD_TOOL_PRIORITY" across all loaded
|
||||
* shared libraries, and if found, store the name of the library
|
||||
*
|
||||
* @param[in]: info A dl_phdr_info struct pointer, which contains information
|
||||
* about library's load address, header, and name.
|
||||
*
|
||||
* @param[in]: size integer size of dl_phdr_info struct
|
||||
*
|
||||
* @param[out]: data copy of the data argument to dl_phdr_iterate call
|
||||
*
|
||||
* @retval:: Return 0 on Success. If callback returns a non-zero value,
|
||||
* dl_iterate_phdr() will stop processing, even if there are unprocessed
|
||||
* shared objects.
|
||||
*/
|
||||
|
||||
static int callback(struct dl_phdr_info* info, size_t size, void* data) {
|
||||
std::vector<std::string>* loadedToolsLib = (std::vector<std::string>*)data;
|
||||
assert(loadedToolsLib != nullptr);
|
||||
/*
|
||||
* Check if lib name is not empty and its not a "vdso.so" lib,
|
||||
* The vDSO is a special shared object file that is built into the Linux kernel.
|
||||
* It is not a regular shared library and thus does not have all the properties
|
||||
* of regular shared libraries. The way the vDSO is loaded and organized in memory
|
||||
* is different from regular shared libraries and it's not guaranteed that it
|
||||
* will have a specific segment or section. Hence its skipped.
|
||||
*/
|
||||
|
||||
if ((info) && (info->dlpi_name[0] != '\0')) {
|
||||
if (std::string(info->dlpi_name).find("vdso.so") != std::string::npos) return 0;
|
||||
|
||||
/*
|
||||
* Iterate through the program headers of the loaded lib and check for PT_DYNAMIC program
|
||||
* header. If the PT_DYNAMIC program header is found, use dlpi_addr and dlpi_phdr members
|
||||
* of dl_phdr_info struct to get the address of the dynamic section of the loaded
|
||||
* library in memory
|
||||
*/
|
||||
|
||||
for (int i = 0; i < info->dlpi_phnum; i++) {
|
||||
if (info->dlpi_phdr[i].p_type == PT_DYNAMIC) {
|
||||
Elf64_Dyn* dyn_section = (Elf64_Dyn*)(info->dlpi_addr + info->dlpi_phdr[i].p_vaddr);
|
||||
|
||||
char* strings = nullptr;
|
||||
Elf64_Xword limit = 0;
|
||||
|
||||
/*
|
||||
* The dynamic section is searched for DT_STRTAB (address of string table),
|
||||
* and DT_STRSZ (size of string table)
|
||||
* DT_NULL - Marks the end of the _DYNAMIC array
|
||||
*/
|
||||
|
||||
for (int j = 0;; j++) {
|
||||
if (dyn_section[j].d_tag == DT_NULL) break;
|
||||
|
||||
if (dyn_section[j].d_tag == DT_STRTAB) strings = (char*)(dyn_section[j].d_un.d_ptr);
|
||||
|
||||
if (dyn_section[j].d_tag == DT_STRSZ) limit = dyn_section[j].d_un.d_val;
|
||||
}
|
||||
|
||||
if (strings == nullptr) debug_print("String table not found");
|
||||
|
||||
/*
|
||||
* Hacky lookup, if string and symbol tables are found,
|
||||
* iterate through the strings in string table and check if
|
||||
* any string matches "HSA_AMD_TOOL_PRIORITY".
|
||||
* If yes, then add the name of the library to the vector of
|
||||
* lib names
|
||||
*/
|
||||
if (strings != nullptr) {
|
||||
char* end = strings + limit;
|
||||
while (strings < end) {
|
||||
if (strcmp(strings, "HSA_AMD_TOOL_PRIORITY") == 0) {
|
||||
loadedToolsLib->push_back(info->dlpi_name);
|
||||
return 0;
|
||||
}
|
||||
strings += (strlen(strings) + 1);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
std::vector<LibHandle> GetLoadedToolsLib() {
|
||||
std::vector<LibHandle> ret;
|
||||
std::vector<std::string> names;
|
||||
|
||||
/* Iterate through all of the loaded shared libraries in the process */
|
||||
dl_iterate_phdr(callback, &names);
|
||||
|
||||
if (!names.empty()) {
|
||||
for (auto& name : names) ret.push_back(LoadLib(name));
|
||||
}
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
std::string GetLibraryName(LibHandle lib) {
|
||||
link_map *map;
|
||||
if(dlinfo(lib, RTLD_DI_LINKMAP, &map)!=0)
|
||||
return "";
|
||||
return map->l_name;
|
||||
}
|
||||
|
||||
Semaphore CreateSemaphore() {
|
||||
sem_t *sem = new sem_t;
|
||||
sem_init(sem, 0, 0);
|
||||
return *(Semaphore*)&sem;
|
||||
}
|
||||
|
||||
bool WaitSemaphore(Semaphore sem) {
|
||||
while(sem_wait(*(sem_t**)&sem))
|
||||
if (errno != EINTR) return false;
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
void PostSemaphore(Semaphore sem) {
|
||||
if (sem_post(*(sem_t**)&sem))
|
||||
assert(false && "Failed to post semaphore");
|
||||
}
|
||||
|
||||
void DestroySemaphore(Semaphore sem) {
|
||||
sem_destroy(*(sem_t**)&sem);
|
||||
delete *(sem_t**)&sem;
|
||||
}
|
||||
|
||||
Mutex CreateMutex() {
|
||||
pthread_mutex_t* mutex = new pthread_mutex_t;
|
||||
pthread_mutex_init(mutex, NULL);
|
||||
return *(Mutex*)&mutex;
|
||||
}
|
||||
|
||||
bool TryAcquireMutex(Mutex lock) {
|
||||
return pthread_mutex_trylock(*(pthread_mutex_t**)&lock) == 0;
|
||||
}
|
||||
|
||||
bool AcquireMutex(Mutex lock) {
|
||||
return pthread_mutex_lock(*(pthread_mutex_t**)&lock) == 0;
|
||||
}
|
||||
|
||||
void ReleaseMutex(Mutex lock) {
|
||||
pthread_mutex_unlock(*(pthread_mutex_t**)&lock);
|
||||
}
|
||||
|
||||
void DestroyMutex(Mutex lock) {
|
||||
pthread_mutex_destroy(*(pthread_mutex_t**)&lock);
|
||||
delete *(pthread_mutex_t**)&lock;
|
||||
}
|
||||
|
||||
void Sleep(int delay_in_millisec) { usleep(delay_in_millisec * 1000); }
|
||||
|
||||
void uSleep(int delayInUs) { usleep(delayInUs); }
|
||||
|
||||
void YieldThread() { sched_yield(); }
|
||||
|
||||
Thread CreateThread(ThreadEntry function, void* threadArgument, uint stackSize) {
|
||||
os_thread* result = new os_thread(function, threadArgument, stackSize);
|
||||
if (!result->Valid()) {
|
||||
delete result;
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
return reinterpret_cast<Thread>(result);
|
||||
}
|
||||
|
||||
void CloseThread(Thread thread) { delete reinterpret_cast<os_thread*>(thread); }
|
||||
|
||||
bool WaitForThread(Thread thread) { return reinterpret_cast<os_thread*>(thread)->Wait(); }
|
||||
|
||||
bool WaitForAllThreads(Thread* threads, uint threadCount) {
|
||||
for (uint i = 0; i < threadCount; i++) WaitForThread(threads[i]);
|
||||
return true;
|
||||
}
|
||||
|
||||
bool IsEnvVarSet(std::string env_var_name) {
|
||||
char* buff = NULL;
|
||||
buff = getenv(env_var_name.c_str());
|
||||
return (buff != NULL);
|
||||
}
|
||||
|
||||
void SetEnvVar(std::string env_var_name, std::string env_var_value) {
|
||||
setenv(env_var_name.c_str(), env_var_value.c_str(), 1);
|
||||
}
|
||||
|
||||
int GetProcessId() {
|
||||
return ::getpid();
|
||||
}
|
||||
|
||||
std::string GetEnvVar(std::string env_var_name) {
|
||||
char* buff;
|
||||
buff = getenv(env_var_name.c_str());
|
||||
std::string ret;
|
||||
if (buff) {
|
||||
ret = buff;
|
||||
}
|
||||
return ret;
|
||||
}
|
||||
|
||||
size_t GetUserModeVirtualMemorySize() {
|
||||
#ifdef _LP64
|
||||
// https://www.kernel.org/doc/Documentation/x86/x86_64/mm.txt :
|
||||
// user space is 0000000000000000 - 00007fffffffffff (=47 bits)
|
||||
return (size_t)(0x800000000000);
|
||||
#else
|
||||
return (size_t)(0xffffffff); // ~4GB
|
||||
#endif
|
||||
}
|
||||
|
||||
size_t GetUsablePhysicalHostMemorySize() {
|
||||
struct sysinfo info = {0};
|
||||
if (sysinfo(&info) != 0) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
const size_t physical_size =
|
||||
static_cast<size_t>(info.totalram * info.mem_unit);
|
||||
return std::min(GetUserModeVirtualMemorySize(), physical_size);
|
||||
}
|
||||
|
||||
uintptr_t GetUserModeVirtualMemoryBase() { return (uintptr_t)0; }
|
||||
|
||||
// Os event implementation
|
||||
typedef struct EventDescriptor_ {
|
||||
pthread_cond_t event;
|
||||
pthread_mutex_t mutex;
|
||||
bool state;
|
||||
bool auto_reset;
|
||||
} EventDescriptor;
|
||||
|
||||
EventHandle CreateOsEvent(bool auto_reset, bool init_state) {
|
||||
EventDescriptor* eventDescrp;
|
||||
eventDescrp = (EventDescriptor*)malloc(sizeof(EventDescriptor));
|
||||
|
||||
pthread_mutex_init(&eventDescrp->mutex, NULL);
|
||||
pthread_cond_init(&eventDescrp->event, NULL);
|
||||
eventDescrp->auto_reset = auto_reset;
|
||||
eventDescrp->state = init_state;
|
||||
|
||||
EventHandle handle = reinterpret_cast<EventHandle>(eventDescrp);
|
||||
|
||||
return handle;
|
||||
}
|
||||
|
||||
int DestroyOsEvent(EventHandle event) {
|
||||
if (event == NULL) {
|
||||
return -1;
|
||||
}
|
||||
|
||||
EventDescriptor* eventDescrp = reinterpret_cast<EventDescriptor*>(event);
|
||||
int ret_code = pthread_cond_destroy(&eventDescrp->event);
|
||||
ret_code |= pthread_mutex_destroy(&eventDescrp->mutex);
|
||||
free(eventDescrp);
|
||||
return ret_code;
|
||||
}
|
||||
|
||||
int WaitForOsEvent(EventHandle event, unsigned int milli_seconds) {
|
||||
if (event == NULL) {
|
||||
return -1;
|
||||
}
|
||||
|
||||
EventDescriptor* eventDescrp = reinterpret_cast<EventDescriptor*>(event);
|
||||
// Event wait time is 0 and state is non-signaled, return directly
|
||||
if (milli_seconds == 0) {
|
||||
int tmp_ret = pthread_mutex_trylock(&eventDescrp->mutex);
|
||||
if (tmp_ret == EBUSY) {
|
||||
// Timeout
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
|
||||
int ret_code = 0;
|
||||
pthread_mutex_lock(&eventDescrp->mutex);
|
||||
if (!eventDescrp->state) {
|
||||
if (milli_seconds == 0) {
|
||||
ret_code = 1;
|
||||
} else {
|
||||
struct timespec ts;
|
||||
struct timeval tp;
|
||||
|
||||
ret_code = gettimeofday(&tp, NULL);
|
||||
ts.tv_sec = tp.tv_sec;
|
||||
ts.tv_nsec = tp.tv_usec * 1000;
|
||||
|
||||
unsigned int sec = milli_seconds / 1000;
|
||||
unsigned int mSec = milli_seconds % 1000;
|
||||
|
||||
ts.tv_sec += sec;
|
||||
ts.tv_nsec += mSec * 1000000;
|
||||
|
||||
// More then one second, add 1 sec to the tv_sec elem
|
||||
if (ts.tv_nsec > 1000000000) {
|
||||
ts.tv_sec += 1;
|
||||
ts.tv_nsec = ts.tv_nsec - 1000000000;
|
||||
}
|
||||
|
||||
ret_code =
|
||||
pthread_cond_timedwait(&eventDescrp->event, &eventDescrp->mutex, &ts);
|
||||
// Time out
|
||||
if (ret_code == 110) {
|
||||
ret_code = 0x14003; // 1 means time out in HSA
|
||||
}
|
||||
|
||||
if (ret_code == 0 && eventDescrp->auto_reset) {
|
||||
eventDescrp->state = false;
|
||||
}
|
||||
}
|
||||
} else if (eventDescrp->auto_reset) {
|
||||
eventDescrp->state = false;
|
||||
}
|
||||
pthread_mutex_unlock(&eventDescrp->mutex);
|
||||
|
||||
return ret_code;
|
||||
}
|
||||
|
||||
int SetOsEvent(EventHandle event) {
|
||||
if (event == NULL) {
|
||||
return -1;
|
||||
}
|
||||
|
||||
EventDescriptor* eventDescrp = reinterpret_cast<EventDescriptor*>(event);
|
||||
int ret_code = 0;
|
||||
ret_code = pthread_mutex_lock(&eventDescrp->mutex);
|
||||
eventDescrp->state = true;
|
||||
ret_code = pthread_mutex_unlock(&eventDescrp->mutex);
|
||||
ret_code |= pthread_cond_signal(&eventDescrp->event);
|
||||
|
||||
return ret_code;
|
||||
}
|
||||
|
||||
int ResetOsEvent(EventHandle event) {
|
||||
if (event == NULL) {
|
||||
return -1;
|
||||
}
|
||||
|
||||
EventDescriptor* eventDescrp = reinterpret_cast<EventDescriptor*>(event);
|
||||
int ret_code = 0;
|
||||
ret_code = pthread_mutex_lock(&eventDescrp->mutex);
|
||||
eventDescrp->state = false;
|
||||
ret_code = pthread_mutex_unlock(&eventDescrp->mutex);
|
||||
|
||||
return ret_code;
|
||||
}
|
||||
|
||||
static double invPeriod = 0.0;
|
||||
|
||||
uint64_t ReadAccurateClock() {
|
||||
if (invPeriod == 0.0) AccurateClockFrequency();
|
||||
timespec time;
|
||||
int err = clock_gettime(CLOCK_MONOTONIC_RAW, &time);
|
||||
if (err != 0) {
|
||||
perror("clock_gettime(CLOCK_MONOTONIC_RAW,...) failed");
|
||||
abort();
|
||||
}
|
||||
return (uint64_t(time.tv_sec) * 1000000000ull + uint64_t(time.tv_nsec)) * invPeriod;
|
||||
}
|
||||
|
||||
uint64_t AccurateClockFrequency() {
|
||||
static clockid_t clock = CLOCK_MONOTONIC;
|
||||
static std::atomic<bool> first(true);
|
||||
// Check kernel version - not a concurrency concern.
|
||||
// use non-RAW for getres due to bug in older 2.6.x kernels
|
||||
if (first.load(std::memory_order_acquire)) {
|
||||
utsname kernelInfo;
|
||||
if (uname(&kernelInfo) == 0) {
|
||||
try {
|
||||
std::string ver = kernelInfo.release;
|
||||
size_t idx;
|
||||
int major = std::stoi(ver, &idx);
|
||||
int minor = std::stoi(ver.substr(idx + 1));
|
||||
if ((major >= 4) && (minor >= 4)) {
|
||||
clock = CLOCK_MONOTONIC_RAW;
|
||||
}
|
||||
} catch (...) {
|
||||
// Kernel version string doesn't conform to the standard pattern.
|
||||
// Keep using the "safe" (non-RAW) clock.
|
||||
}
|
||||
}
|
||||
first.store(false, std::memory_order_release);
|
||||
}
|
||||
timespec time;
|
||||
int err = clock_getres(clock, &time);
|
||||
if (err != 0) {
|
||||
perror("clock_getres failed");
|
||||
abort();
|
||||
}
|
||||
if (time.tv_sec != 0 || time.tv_nsec >= 0xFFFFFFFF) {
|
||||
fprintf(stderr,
|
||||
"clock_getres(CLOCK_MONOTONIC(_RAW),...) returned very low "
|
||||
"frequency (<1Hz).\n");
|
||||
abort();
|
||||
}
|
||||
if (invPeriod == 0.0) invPeriod = 1.0 / double(time.tv_nsec);
|
||||
return 1000000000ull / uint64_t(time.tv_nsec);
|
||||
}
|
||||
|
||||
SharedMutex CreateSharedMutex() {
|
||||
pthread_rwlockattr_t attrib;
|
||||
int err = pthread_rwlockattr_init(&attrib);
|
||||
if (err != 0) {
|
||||
fprintf(stderr, "rw lock attribute init failed: %s\n", strerror(err));
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
#ifdef __GLIBC__
|
||||
err = pthread_rwlockattr_setkind_np(&attrib, PTHREAD_RWLOCK_PREFER_WRITER_NONRECURSIVE_NP);
|
||||
if (err != 0) {
|
||||
fprintf(stderr, "Set rw lock attribute failure: %s\n", strerror(err));
|
||||
return nullptr;
|
||||
}
|
||||
#else
|
||||
err = pthread_rwlockattr_setkind(&attrib, PTHREAD_RWLOCK_PREFER_WRITER_NONRECURSIVE_NP);
|
||||
if (err != 0) {
|
||||
fprintf(stderr, "Set rw lock attribute failure: %s\n", strerror(err));
|
||||
return nullptr;
|
||||
}
|
||||
#endif
|
||||
|
||||
pthread_rwlock_t* lock = new pthread_rwlock_t;
|
||||
err = pthread_rwlock_init(lock, &attrib);
|
||||
if (err != 0) {
|
||||
fprintf(stderr, "rw lock init failed: %s\n", strerror(err));
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
pthread_rwlockattr_destroy(&attrib);
|
||||
return lock;
|
||||
}
|
||||
|
||||
bool TryAcquireSharedMutex(SharedMutex lock) {
|
||||
int err = pthread_rwlock_trywrlock(*(pthread_rwlock_t**)&lock);
|
||||
return err == 0;
|
||||
}
|
||||
|
||||
bool AcquireSharedMutex(SharedMutex lock) {
|
||||
int err = pthread_rwlock_wrlock(*(pthread_rwlock_t**)&lock);
|
||||
return err == 0;
|
||||
}
|
||||
|
||||
void ReleaseSharedMutex(SharedMutex lock) {
|
||||
int err = pthread_rwlock_unlock(*(pthread_rwlock_t**)&lock);
|
||||
if (err != 0) {
|
||||
fprintf(stderr, "SharedMutex unlock failed: %s\n", strerror(err));
|
||||
abort();
|
||||
}
|
||||
}
|
||||
|
||||
bool TrySharedAcquireSharedMutex(SharedMutex lock) {
|
||||
int err = pthread_rwlock_tryrdlock(*(pthread_rwlock_t**)&lock);
|
||||
return err == 0;
|
||||
}
|
||||
|
||||
bool SharedAcquireSharedMutex(SharedMutex lock) {
|
||||
int err = pthread_rwlock_rdlock(*(pthread_rwlock_t**)&lock);
|
||||
return err == 0;
|
||||
}
|
||||
|
||||
void SharedReleaseSharedMutex(SharedMutex lock) {
|
||||
int err = pthread_rwlock_unlock(*(pthread_rwlock_t**)&lock);
|
||||
if (err != 0) {
|
||||
fprintf(stderr, "SharedMutex unlock failed: %s\n", strerror(err));
|
||||
abort();
|
||||
}
|
||||
}
|
||||
|
||||
void DestroySharedMutex(SharedMutex lock) {
|
||||
pthread_rwlock_destroy(*(pthread_rwlock_t**)&lock);
|
||||
delete *(pthread_rwlock_t**)&lock;
|
||||
}
|
||||
|
||||
static uint64_t sys_clock_period_ = 0;
|
||||
|
||||
uint64_t ReadSystemClock() {
|
||||
struct timespec ts;
|
||||
clock_gettime(CLOCK_BOOTTIME, &ts);
|
||||
uint64_t time = (uint64_t(ts.tv_sec) * 1000000000 + uint64_t(ts.tv_nsec));
|
||||
if (sys_clock_period_ != 1)
|
||||
return time / sys_clock_period_;
|
||||
else
|
||||
return time;
|
||||
}
|
||||
|
||||
uint64_t SystemClockFrequency() {
|
||||
struct timespec ts;
|
||||
clock_getres(CLOCK_BOOTTIME, &ts);
|
||||
sys_clock_period_ = (uint64_t(ts.tv_sec) * 1000000000 + uint64_t(ts.tv_nsec));
|
||||
return 1000000000 / sys_clock_period_;
|
||||
}
|
||||
|
||||
bool ParseCpuID(cpuid_t* cpuinfo) {
|
||||
#if defined(__i386__) || defined(__x86_64__)
|
||||
uint32_t eax, ebx, ecx, edx, max_eax = 0;
|
||||
memset(cpuinfo, 0, sizeof(*cpuinfo));
|
||||
|
||||
/* Make sure current CPU supports at least EAX 4 */
|
||||
if (!__get_cpuid_max(0x80000004, NULL)) return false;
|
||||
|
||||
// Manufacturer ID is a twelve-character ASCII string stored in order EBX, EDX, ECX.
|
||||
if (!__get_cpuid(0, &max_eax, (uint32_t*)&cpuinfo->ManufacturerID[0],
|
||||
(uint32_t*)&cpuinfo->ManufacturerID[8],
|
||||
(uint32_t*)&cpuinfo->ManufacturerID[4])) {
|
||||
return false;
|
||||
}
|
||||
|
||||
if (!strcmp(cpuinfo->ManufacturerID, "AuthenticAMD")) {
|
||||
if (__get_cpuid(0x80000001, &eax, &ebx, &ecx, &edx)) {
|
||||
cpuinfo->mwaitx = !!((ecx >> 29) & 0x1);
|
||||
}
|
||||
}
|
||||
return true;
|
||||
#else
|
||||
return false;
|
||||
#endif
|
||||
}
|
||||
|
||||
} // namespace os
|
||||
} // namespace rocr
|
||||
|
||||
#endif
|
||||
@@ -0,0 +1,290 @@
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
//
|
||||
// The University of Illinois/NCSA
|
||||
// Open Source License (NCSA)
|
||||
//
|
||||
// Copyright (c) 2014-2020, Advanced Micro Devices, Inc. All rights reserved.
|
||||
//
|
||||
// Developed by:
|
||||
//
|
||||
// AMD Research and AMD HSA Software Development
|
||||
//
|
||||
// Advanced Micro Devices, Inc.
|
||||
//
|
||||
// www.amd.com
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
// of this software and associated documentation files (the "Software"), to
|
||||
// deal with the Software without restriction, including without limitation
|
||||
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
// and/or sell copies of the Software, and to permit persons to whom the
|
||||
// Software is furnished to do so, subject to the following conditions:
|
||||
//
|
||||
// - Redistributions of source code must retain the above copyright notice,
|
||||
// this list of conditions and the following disclaimers.
|
||||
// - Redistributions in binary form must reproduce the above copyright
|
||||
// notice, this list of conditions and the following disclaimers in
|
||||
// the documentation and/or other materials provided with the distribution.
|
||||
// - Neither the names of Advanced Micro Devices, Inc,
|
||||
// nor the names of its contributors may be used to endorse or promote
|
||||
// products derived from this Software without specific prior written
|
||||
// permission.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
|
||||
// THE CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
|
||||
// OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
|
||||
// ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
||||
// DEALINGS WITH THE SOFTWARE.
|
||||
//
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
// Library of syncronization primitives - to be added to as needed.
|
||||
|
||||
#ifndef HSA_RUNTIME_CORE_UTIL_LOCKS_H_
|
||||
#define HSA_RUNTIME_CORE_UTIL_LOCKS_H_
|
||||
|
||||
#include "utils.h"
|
||||
#include "os.h"
|
||||
|
||||
namespace rocr {
|
||||
|
||||
class HybridMutex {
|
||||
public:
|
||||
HybridMutex():lock_(0) {
|
||||
sem_ = os::CreateSemaphore();
|
||||
}
|
||||
|
||||
~HybridMutex() {
|
||||
os::DestroySemaphore(sem_);
|
||||
}
|
||||
|
||||
bool Try() {
|
||||
int old = 0;
|
||||
return lock_.compare_exchange_strong(old, 1);
|
||||
}
|
||||
|
||||
bool Acquire() {
|
||||
int cnt = maxSpinIterPause + maxSpinIterYield;
|
||||
|
||||
int old = 0;
|
||||
while (!lock_.compare_exchange_strong(old, 1)) {
|
||||
cnt--;
|
||||
if (cnt > maxSpinIterPause) {
|
||||
_mm_pause();
|
||||
} else if (cnt-- > maxSpinIterYield) {
|
||||
os::YieldThread();
|
||||
} else {
|
||||
os::WaitSemaphore(sem_);
|
||||
cnt = maxSpinIterPause + maxSpinIterYield;
|
||||
}
|
||||
old = 0;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
void Release() {
|
||||
int old = 1;
|
||||
if (lock_.compare_exchange_strong(old, 0))
|
||||
os::PostSemaphore(sem_);
|
||||
}
|
||||
|
||||
private:
|
||||
std::atomic<int> lock_;
|
||||
os::Semaphore sem_;
|
||||
const uint32_t maxSpinIterPause = 55;
|
||||
const uint32_t maxSpinIterYield = 55;
|
||||
|
||||
/// @brief: Disable copiable and assignable ability.
|
||||
DISALLOW_COPY_AND_ASSIGN(HybridMutex);
|
||||
};
|
||||
|
||||
|
||||
/// @brief: a class represents a kernel mutex.
|
||||
/// Uses the kernel's scheduler to keep the waiting thread from being scheduled
|
||||
/// until the lock is released (Best for long waits, though anything using
|
||||
/// a kernel object is a long wait).
|
||||
class KernelMutex {
|
||||
public:
|
||||
KernelMutex() { lock_ = os::CreateMutex(); }
|
||||
~KernelMutex() { os::DestroyMutex(lock_); }
|
||||
|
||||
bool Try() { return os::TryAcquireMutex(lock_); }
|
||||
bool Acquire() { return os::AcquireMutex(lock_); }
|
||||
void Release() { os::ReleaseMutex(lock_); }
|
||||
|
||||
private:
|
||||
os::Mutex lock_;
|
||||
|
||||
/// @brief: Disable copiable and assignable ability.
|
||||
DISALLOW_COPY_AND_ASSIGN(KernelMutex);
|
||||
};
|
||||
|
||||
/// @brief: represents a spin lock.
|
||||
/// For very short hold durations on the order of the thread scheduling
|
||||
/// quanta or less.
|
||||
class SpinMutex {
|
||||
public:
|
||||
SpinMutex() { lock_ = 0; }
|
||||
|
||||
bool Try() {
|
||||
int old = 0;
|
||||
return lock_.compare_exchange_strong(old, 1);
|
||||
}
|
||||
bool Acquire() {
|
||||
int old = 0;
|
||||
while (!lock_.compare_exchange_strong(old, 1))
|
||||
{
|
||||
old=0;
|
||||
os::YieldThread();
|
||||
}
|
||||
return true;
|
||||
}
|
||||
void Release() { lock_ = 0; }
|
||||
|
||||
private:
|
||||
std::atomic<int> lock_;
|
||||
|
||||
/// @brief: Disable copiable and assignable ability.
|
||||
DISALLOW_COPY_AND_ASSIGN(SpinMutex);
|
||||
};
|
||||
|
||||
class KernelEvent {
|
||||
public:
|
||||
KernelEvent() { evt_ = os::CreateOsEvent(true, true); }
|
||||
~KernelEvent() { os::DestroyOsEvent(evt_); }
|
||||
|
||||
bool IsSet() { return os::WaitForOsEvent(evt_, 0)==0; }
|
||||
bool WaitForSet() { return os::WaitForOsEvent(evt_, 0xFFFFFFFF)==0; }
|
||||
void Set() { os::SetOsEvent(evt_); }
|
||||
void Reset() { os::ResetOsEvent(evt_); }
|
||||
|
||||
private:
|
||||
os::EventHandle evt_;
|
||||
|
||||
/// @brief: Disable copiable and assignable ability.
|
||||
DISALLOW_COPY_AND_ASSIGN(KernelEvent);
|
||||
};
|
||||
|
||||
/// @brief: represents a yielding shared mutex.
|
||||
/// aka read/write mutex
|
||||
class KernelSharedMutex {
|
||||
public:
|
||||
/// @brief: Interfaces ScopedAcquire to shared operations.
|
||||
class Shared {
|
||||
public:
|
||||
explicit Shared(KernelSharedMutex* lock) : lock_(lock) {}
|
||||
bool Try() { return lock_->TryShared(); }
|
||||
bool Acquire() { return lock_->AcquireShared(); }
|
||||
void Release() { lock_->ReleaseShared(); }
|
||||
|
||||
private:
|
||||
KernelSharedMutex* lock_;
|
||||
};
|
||||
|
||||
KernelSharedMutex() { lock_ = os::CreateSharedMutex(); }
|
||||
~KernelSharedMutex() { os::DestroySharedMutex(lock_); }
|
||||
|
||||
// Exclusive mode operations
|
||||
bool Try() { return os::TryAcquireSharedMutex(lock_); }
|
||||
bool Acquire() { return os::AcquireSharedMutex(lock_); }
|
||||
void Release() { os::ReleaseSharedMutex(lock_); }
|
||||
|
||||
// Shared mode operations
|
||||
bool TryShared() { return os::TrySharedAcquireSharedMutex(lock_); }
|
||||
bool AcquireShared() { return os::SharedAcquireSharedMutex(lock_); }
|
||||
void ReleaseShared() { os::SharedReleaseSharedMutex(lock_); }
|
||||
|
||||
// Return shared operations interface
|
||||
Shared shared() { return Shared(this); }
|
||||
|
||||
private:
|
||||
os::SharedMutex lock_;
|
||||
|
||||
/// @brief: Disable copiable and assignable ability.
|
||||
DISALLOW_COPY_AND_ASSIGN(KernelSharedMutex);
|
||||
};
|
||||
|
||||
/// @brief: Type trait to identify mutex types
|
||||
template <class T> class isMutex {
|
||||
public:
|
||||
enum { value = false };
|
||||
};
|
||||
template <> class isMutex<HybridMutex> {
|
||||
public:
|
||||
enum { value = true };
|
||||
};
|
||||
template <> class isMutex<KernelMutex> {
|
||||
public:
|
||||
enum { value = true };
|
||||
};
|
||||
template <> class isMutex<SpinMutex> {
|
||||
public:
|
||||
enum { value = true };
|
||||
};
|
||||
template <> class isMutex<KernelSharedMutex> {
|
||||
public:
|
||||
enum { value = true };
|
||||
};
|
||||
|
||||
/// @brief: A class behaves as a lock in a scope. When trying to enter into the
|
||||
/// critical section, creat a object of this class. After the control path goes
|
||||
/// out of the scope, it will release the lock automatically.
|
||||
template <class LockType> class ScopedAcquire {
|
||||
public:
|
||||
/// @brief: When constructing, acquire the lock.
|
||||
/// @param: lock(Input), pointer to an existing lock.
|
||||
explicit ScopedAcquire(LockType* lock) : lock_(lock), doRelease(true) {
|
||||
static_assert(isMutex<LockType>::value, "ScopedAcquire requires a mutex type.");
|
||||
lock_.Acquire();
|
||||
}
|
||||
explicit ScopedAcquire(LockType lock) : lock_(lock), doRelease(true) {
|
||||
static_assert(!isMutex<LockType>::value, "Mutex types are not copyable.");
|
||||
lock_.Acquire();
|
||||
}
|
||||
|
||||
/// @brief: when destructing, release the lock.
|
||||
~ScopedAcquire() {
|
||||
if (doRelease) lock_.Release();
|
||||
}
|
||||
|
||||
/// @brief: Release the lock early. Avoid using when possible.
|
||||
void Release() {
|
||||
lock_.Release();
|
||||
doRelease = false;
|
||||
}
|
||||
|
||||
private:
|
||||
/// @brief: Adapts between pointers to mutex types and mutex pointer types.
|
||||
template <class T, bool B> class container {
|
||||
public:
|
||||
container(T* lock) : lock_(lock) {}
|
||||
__forceinline bool Acquire() { return lock_->Acquire(); }
|
||||
__forceinline void Release() { return lock_->Release(); }
|
||||
|
||||
private:
|
||||
T* lock_;
|
||||
};
|
||||
|
||||
/// @brief: Specialization for mutex pointer types.
|
||||
template <class T> class container<T, false> {
|
||||
public:
|
||||
container(T lock) : lock_(lock) {}
|
||||
__forceinline bool Acquire() { return lock_.Acquire(); }
|
||||
__forceinline void Release() { return lock_.Release(); }
|
||||
|
||||
private:
|
||||
T lock_;
|
||||
};
|
||||
|
||||
container<LockType, isMutex<LockType>::value> lock_;
|
||||
bool doRelease;
|
||||
|
||||
/// @brief: Disable copiable and assignable ability.
|
||||
DISALLOW_COPY_AND_ASSIGN(ScopedAcquire);
|
||||
};
|
||||
|
||||
} // namespace rocr
|
||||
|
||||
#endif // HSA_RUNTIME_CORE_SUTIL_LOCKS_H_
|
||||
+327
@@ -0,0 +1,327 @@
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
//
|
||||
// The University of Illinois/NCSA
|
||||
// Open Source License (NCSA)
|
||||
//
|
||||
// Copyright (c) 2014-2024, Advanced Micro Devices, Inc. All rights reserved.
|
||||
//
|
||||
// Developed by:
|
||||
//
|
||||
// AMD Research and AMD HSA Software Development
|
||||
//
|
||||
// Advanced Micro Devices, Inc.
|
||||
//
|
||||
// www.amd.com
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
// of this software and associated documentation files (the "Software"), to
|
||||
// deal with the Software without restriction, including without limitation
|
||||
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
// and/or sell copies of the Software, and to permit persons to whom the
|
||||
// Software is furnished to do so, subject to the following conditions:
|
||||
//
|
||||
// - Redistributions of source code must retain the above copyright notice,
|
||||
// this list of conditions and the following disclaimers.
|
||||
// - Redistributions in binary form must reproduce the above copyright
|
||||
// notice, this list of conditions and the following disclaimers in
|
||||
// the documentation and/or other materials provided with the distribution.
|
||||
// - Neither the names of Advanced Micro Devices, Inc,
|
||||
// nor the names of its contributors may be used to endorse or promote
|
||||
// products derived from this Software without specific prior written
|
||||
// permission.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
|
||||
// THE CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
|
||||
// OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
|
||||
// ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
||||
// DEALINGS WITH THE SOFTWARE.
|
||||
//
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
// Minimal operating system abstraction interfaces.
|
||||
|
||||
#ifndef HSA_RUNTIME_CORE_UTIL_OS_H_
|
||||
#define HSA_RUNTIME_CORE_UTIL_OS_H_
|
||||
|
||||
#include <string>
|
||||
#include <vector>
|
||||
#include "utils.h"
|
||||
|
||||
namespace rocr {
|
||||
namespace os {
|
||||
typedef void* LibHandle;
|
||||
typedef void* Semaphore;
|
||||
typedef void* Mutex;
|
||||
typedef void* SharedMutex;
|
||||
typedef void* Thread;
|
||||
typedef void* EventHandle;
|
||||
|
||||
enum class os_t { OS_WIN = 0, OS_LINUX, COUNT };
|
||||
static __forceinline std::underlying_type<os_t>::type os_index(os_t val) {
|
||||
return std::underlying_type<os_t>::type(val);
|
||||
}
|
||||
|
||||
#ifdef _WIN32
|
||||
static const os_t current_os = os_t::OS_WIN;
|
||||
#elif __linux__
|
||||
static const os_t current_os = os_t::OS_LINUX;
|
||||
#else
|
||||
static_assert(false, "Operating System not detected!");
|
||||
#endif
|
||||
|
||||
/// @brief: Loads dynamic library based on file name. Return value will be NULL
|
||||
/// if failed.
|
||||
/// @param: filename(Input), file name of the library.
|
||||
/// @return: LibHandle.
|
||||
LibHandle LoadLib(std::string filename);
|
||||
|
||||
/// @brief: Gets the address of exported symbol. Return NULl if failed.
|
||||
/// @param: lib(Input), library handle which exporting from.
|
||||
/// @param: export_name(Input), the name of the exported symbol.
|
||||
/// @return: void*.
|
||||
void* GetExportAddress(LibHandle lib, std::string export_name);
|
||||
|
||||
/// @brief: Unloads the dynamic library.
|
||||
/// @param: lib(Input), library handle which will be unloaded.
|
||||
void CloseLib(LibHandle lib);
|
||||
|
||||
/// @brief: Lists loaded tool libraries that contain
|
||||
/// symbol HSA_AMD_TOOL_PRIORITY
|
||||
/// @return: List of library handles
|
||||
std::vector<LibHandle> GetLoadedToolsLib();
|
||||
|
||||
/// @brief: Returns the library's path name.
|
||||
/// @param: lib(Input), libray handle
|
||||
/// @return: Path name of library
|
||||
std::string GetLibraryName(LibHandle lib);
|
||||
|
||||
/// @brief: Creates a Semaphore, will return NULL if failed.
|
||||
/// @param: void.
|
||||
/// @return: Semaphore.
|
||||
Semaphore CreateSemaphore();
|
||||
|
||||
/// @brief: Waits for the semaphore. This is a blocking wait.
|
||||
/// If the Semaphore is signalled, this function will return.
|
||||
/// @param: sem(Input), handle to the semaphore.
|
||||
/// @return: void.
|
||||
bool WaitSemaphore(Semaphore sem);
|
||||
|
||||
/// @brief: Post/Signal/Wake-up the semaphore
|
||||
/// @param: sem(Input), handle to the semaphore.
|
||||
/// @return: void.
|
||||
void PostSemaphore(Semaphore sem);
|
||||
|
||||
/// @brief: Destroys the semaphore.
|
||||
/// @param: sem(Input), handle to the semaphore.
|
||||
/// @return: void.
|
||||
void DestroySemaphore(Semaphore sem);
|
||||
|
||||
/// @brief: Creates a mutex, will return NULL if failed.
|
||||
/// @param: void.
|
||||
/// @return: Mutex.
|
||||
Mutex CreateMutex();
|
||||
|
||||
/// @brief: Tries to acquire the mutex once, if successed, return true.
|
||||
/// @param: lock(Input), handle to the mutex.
|
||||
/// @return: bool.
|
||||
bool TryAcquireMutex(Mutex lock);
|
||||
|
||||
/// @brief: Aquires the mutex, if the mutex is locked, it will wait until it is
|
||||
/// released. If the mutex is acquired successfully, it will return true.
|
||||
/// @param: lock(Input), handle to the mutex.
|
||||
/// @return: bool.
|
||||
bool AcquireMutex(Mutex lock);
|
||||
|
||||
/// @brief: Releases the mutex.
|
||||
/// @param: lock(Input), handle to the mutex.
|
||||
/// @return: void.
|
||||
void ReleaseMutex(Mutex lock);
|
||||
|
||||
/// @brief: Destroys the mutex.
|
||||
/// @param: lock(Input), handle to the mutex.
|
||||
/// @return: void.
|
||||
void DestroyMutex(Mutex lock);
|
||||
|
||||
/// @brief: Creates a shared mutex, will return NULL if failed.
|
||||
/// @param: void.
|
||||
/// @return: SharedMutex.
|
||||
SharedMutex CreateSharedMutex();
|
||||
|
||||
/// @brief: Tries to acquire the mutex in exclusive mode once, if successed, return true.
|
||||
/// @param: lock(Input), handle to the shared mutex.
|
||||
/// @return: bool.
|
||||
bool TryAcquireSharedMutex(SharedMutex lock);
|
||||
|
||||
/// @brief: Aquires the mutex in exclusive mode, if the mutex is locked, it will wait until it is
|
||||
/// released. If the mutex is acquired successfully, it will return true.
|
||||
/// @param: lock(Input), handle to the mutex.
|
||||
/// @return: bool.
|
||||
bool AcquireSharedMutex(SharedMutex lock);
|
||||
|
||||
/// @brief: Releases the mutex from exclusive mode.
|
||||
/// @param: lock(Input), handle to the mutex.
|
||||
/// @return: void.
|
||||
void ReleaseSharedMutex(SharedMutex lock);
|
||||
|
||||
/// @brief: Tries to acquire the mutex in shared mode once, if successed, return true.
|
||||
/// @param: lock(Input), handle to the mutex.
|
||||
/// @return: bool.
|
||||
bool TrySharedAcquireSharedMutex(SharedMutex lock);
|
||||
|
||||
/// @brief: Aquires the mutex in shared mode, if the mutex in exclusive mode, it will wait until it
|
||||
/// is released. If the mutex is acquired successfully, it will return true.
|
||||
/// @param: lock(Input), handle to the mutex.
|
||||
/// @return: bool.
|
||||
bool SharedAcquireSharedMutex(SharedMutex lock);
|
||||
|
||||
/// @brief: Releases the mutex from shared mode.
|
||||
/// @param: lock(Input), handle to the mutex.
|
||||
/// @return: void.
|
||||
void SharedReleaseSharedMutex(SharedMutex lock);
|
||||
|
||||
/// @brief: Destroys the mutex.
|
||||
/// @param: lock(Input), handle to the mutex.
|
||||
/// @return: void.
|
||||
void DestroySharedMutex(SharedMutex lock);
|
||||
|
||||
/// @brief: Puts current thread to sleep.
|
||||
/// @param: delayInMs(Input), time in millisecond for sleeping.
|
||||
/// @return: void.
|
||||
void Sleep(int delayInMs);
|
||||
|
||||
/// @brief: Puts current thread to sleep.
|
||||
/// @param: delayInMs(Input), time in millisecond for sleeping.
|
||||
/// @return: void.
|
||||
void uSleep(int delayInUs);
|
||||
|
||||
/// @brief: Yields current thread.
|
||||
/// @param: void.
|
||||
/// @return: void.
|
||||
void YieldThread();
|
||||
|
||||
typedef void (*ThreadEntry)(void*);
|
||||
|
||||
/// @brief: Creates a thread will return NULL if failed.
|
||||
/// @param: entry_function(Input), a pointer to the function which the thread
|
||||
/// starts from.
|
||||
/// @param: entry_argument(Input), a pointer to the argument of the thread
|
||||
/// function.
|
||||
/// @param: stack_size(Input), size of the thread's stack, 0 by default.
|
||||
/// @return: Thread, a handle to thread created.
|
||||
Thread CreateThread(ThreadEntry entry_function, void* entry_argument,
|
||||
uint stack_size = 0);
|
||||
|
||||
/// @brief: Destroys the thread.
|
||||
/// @param: thread(Input), thread handle to what will be destroyed.
|
||||
/// @return: void.
|
||||
void CloseThread(Thread thread);
|
||||
|
||||
/// @brief: Waits for specific thread to finish, if successful, return true.
|
||||
/// @param: thread(Input), handle to waiting thread.
|
||||
/// @return: bool.
|
||||
bool WaitForThread(Thread thread);
|
||||
|
||||
/// @brief: Waits for multiple threads to finish, if successful, return true.
|
||||
/// @param; threads(Input), a pointer to a list of thread handle.
|
||||
/// @param: thread_count(Input), number of threads to be waited on.
|
||||
/// @return: bool.
|
||||
bool WaitForAllThreads(Thread* threads, uint thread_count);
|
||||
|
||||
/// @brief: Determines if environment key is set.
|
||||
/// @param: env_var_name(Input), name of the environment value.
|
||||
/// @return: bool, true for binding any value to environment key,
|
||||
/// including an empty string. False otherwise
|
||||
bool IsEnvVarSet(std::string env_var_name);
|
||||
|
||||
/// @brief: Sets the environment value.
|
||||
/// @param: env_var_name(Input), name of the environment value.
|
||||
/// @param: env_var_value(Input), value of the environment value.s
|
||||
/// @return: void.
|
||||
void SetEnvVar(std::string env_var_name, std::string env_var_value);
|
||||
|
||||
/// @brief: Gets the value of environment value.
|
||||
/// @param: env_var_name(Input), name of the environment value.
|
||||
/// @return: std::string, value of the environment value, returned as string.
|
||||
std::string GetEnvVar(std::string env_var_name);
|
||||
|
||||
/// @brief: Gets the process ID.
|
||||
/// @param: void
|
||||
/// @return: int, process ID returned as int.
|
||||
int GetProcessId();
|
||||
|
||||
/// @brief: Gets the max virtual memory size accessible to the application.
|
||||
/// @param: void.
|
||||
/// @return: size_t, size of the accessible memory to the application.
|
||||
size_t GetUserModeVirtualMemorySize();
|
||||
|
||||
/// @brief: Gets the max physical host system memory size.
|
||||
/// @param: void.
|
||||
/// @return: size_t, size of the physical host system memory.
|
||||
size_t GetUsablePhysicalHostMemorySize();
|
||||
|
||||
/// @brief: Gets the virtual memory base address. It is hardcoded to 0.
|
||||
/// @param: void.
|
||||
/// @return: uintptr_t, always 0.
|
||||
uintptr_t GetUserModeVirtualMemoryBase();
|
||||
|
||||
/// @brief os event api, create an event
|
||||
/// @param: auto_reset whether an event can reset the status automatically
|
||||
/// @param: init_state initial state of the event
|
||||
/// @return: event handle
|
||||
EventHandle CreateOsEvent(bool auto_reset, bool init_state);
|
||||
|
||||
/// @brief os event api, destroy an event
|
||||
/// @param: event handle
|
||||
/// @return: whether destroy is correct
|
||||
int DestroyOsEvent(EventHandle event);
|
||||
|
||||
/// @brief os event api, wait on event
|
||||
/// @param: event Event handle
|
||||
/// @param: milli_seconds wait time
|
||||
/// @return: Indicate success or timeout
|
||||
int WaitForOsEvent(EventHandle event, unsigned int milli_seconds);
|
||||
|
||||
/// @brief os event api, set event state
|
||||
/// @param: event Event handle
|
||||
/// @return: Whether event set is correct
|
||||
int SetOsEvent(EventHandle event);
|
||||
|
||||
/// @brief os event api, reset event state
|
||||
/// @param: event Event handle
|
||||
/// @return: Whether event reset is correct
|
||||
int ResetOsEvent(EventHandle event);
|
||||
|
||||
/// @brief reads a clock which is deemed to be accurate for elapsed time
|
||||
/// measurements, though not necessarilly fast to query
|
||||
/// @return clock counter value
|
||||
uint64_t ReadAccurateClock();
|
||||
|
||||
/// @brief retrieves the frequency in Hz of the unit used in ReadAccurateClock.
|
||||
/// It does not necessarilly reflect the resolution of the clock, but is the
|
||||
/// value needed to convert a difference in the clock's counter value to elapsed
|
||||
/// seconds. This frequency does not change at runtime.
|
||||
/// @return returns the frequency
|
||||
uint64_t AccurateClockFrequency();
|
||||
|
||||
/// @brief read the system clock which serves as the HSA system clock
|
||||
/// counter in KFD.
|
||||
uint64_t ReadSystemClock();
|
||||
|
||||
/// @brief read the system clock frequency
|
||||
uint64_t SystemClockFrequency();
|
||||
|
||||
typedef struct cpuid_s {
|
||||
char ManufacturerID[13]; // 12 char, NULL terminated
|
||||
bool mwaitx;
|
||||
} cpuid_t;
|
||||
|
||||
/// @brief parse CPUID
|
||||
/// @param: cpuinfo struct to be filled
|
||||
bool ParseCpuID(cpuid_t* cpuinfo);
|
||||
|
||||
} // namespace os
|
||||
} // namespace rocr
|
||||
|
||||
#endif // HSA_RUNTIME_CORE_UTIL_OS_H_
|
||||
@@ -0,0 +1,363 @@
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
//
|
||||
// The University of Illinois/NCSA
|
||||
// Open Source License (NCSA)
|
||||
//
|
||||
// Copyright (c) 2014-2020, Advanced Micro Devices, Inc. All rights reserved.
|
||||
//
|
||||
// Developed by:
|
||||
//
|
||||
// AMD Research and AMD HSA Software Development
|
||||
//
|
||||
// Advanced Micro Devices, Inc.
|
||||
//
|
||||
// www.amd.com
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
// of this software and associated documentation files (the "Software"), to
|
||||
// deal with the Software without restriction, including without limitation
|
||||
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
// and/or sell copies of the Software, and to permit persons to whom the
|
||||
// Software is furnished to do so, subject to the following conditions:
|
||||
//
|
||||
// - Redistributions of source code must retain the above copyright notice,
|
||||
// this list of conditions and the following disclaimers.
|
||||
// - Redistributions in binary form must reproduce the above copyright
|
||||
// notice, this list of conditions and the following disclaimers in
|
||||
// the documentation and/or other materials provided with the distribution.
|
||||
// - Neither the names of Advanced Micro Devices, Inc,
|
||||
// nor the names of its contributors may be used to endorse or promote
|
||||
// products derived from this Software without specific prior written
|
||||
// permission.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
|
||||
// THE CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
|
||||
// OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
|
||||
// ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
||||
// DEALINGS WITH THE SOFTWARE.
|
||||
//
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
// A simple best fit memory allocator with eager compaction. Manages block sub-allocation.
|
||||
// For use when memory efficiency is more important than allocation speed.
|
||||
// O(log n) time.
|
||||
|
||||
#ifndef HSA_RUNTME_CORE_UTIL_SIMPLE_HEAP_H_
|
||||
#define HSA_RUNTME_CORE_UTIL_SIMPLE_HEAP_H_
|
||||
|
||||
#include <map>
|
||||
#include <deque>
|
||||
#include <utility>
|
||||
|
||||
#include "core/util/utils.h"
|
||||
|
||||
namespace rocr {
|
||||
|
||||
template <typename Allocator> class SimpleHeap {
|
||||
private:
|
||||
struct Fragment_T {
|
||||
typedef std::multimap<size_t, uintptr_t>::iterator ptr_t;
|
||||
ptr_t free_list_entry_;
|
||||
struct {
|
||||
size_t size : 62;
|
||||
bool discard : 1;
|
||||
bool free : 1;
|
||||
};
|
||||
|
||||
Fragment_T(ptr_t Iterator, size_t Len, bool Free)
|
||||
: free_list_entry_(Iterator), size(Len), discard(false), free(Free) {}
|
||||
Fragment_T() = default;
|
||||
};
|
||||
|
||||
struct Block {
|
||||
uintptr_t base_ptr_;
|
||||
size_t length_;
|
||||
|
||||
Block(uintptr_t base, size_t length) : base_ptr_(base), length_(length) {}
|
||||
Block() = default;
|
||||
};
|
||||
|
||||
Allocator block_allocator_;
|
||||
|
||||
std::multimap<size_t, uintptr_t> free_list_;
|
||||
std::map<uintptr_t, std::map<uintptr_t, Fragment_T>> block_list_;
|
||||
std::deque<Block> block_cache_;
|
||||
|
||||
// Size of blocks that are at least partially in use.
|
||||
size_t in_use_size_;
|
||||
// Total size of block cache
|
||||
size_t cache_size_;
|
||||
|
||||
__forceinline bool isFree(const Fragment_T& node) { return node.free; }
|
||||
__forceinline void setUsed(Fragment_T& node) {
|
||||
node.free = false;
|
||||
node.free_list_entry_ = free_list_.end();
|
||||
}
|
||||
__forceinline void setFree(Fragment_T& node, typename Fragment_T::ptr_t Iterator) {
|
||||
node.free_list_entry_ = Iterator;
|
||||
node.free = true;
|
||||
}
|
||||
__forceinline Fragment_T makeFragment(size_t Len) {
|
||||
return Fragment_T(free_list_.end(), Len, false);
|
||||
}
|
||||
__forceinline Fragment_T makeFragment(typename Fragment_T::ptr_t Iterator, size_t Len) {
|
||||
return Fragment_T(Iterator, Len, true);
|
||||
}
|
||||
__forceinline void removeFreeListEntry(Fragment_T& node) {
|
||||
if (node.free_list_entry_ != free_list_.end()) {
|
||||
free_list_.erase(node.free_list_entry_);
|
||||
node.free_list_entry_ = free_list_.end();
|
||||
}
|
||||
}
|
||||
__forceinline void discard(Fragment_T& node) {
|
||||
removeFreeListEntry(node);
|
||||
node.discard = true;
|
||||
}
|
||||
|
||||
public:
|
||||
explicit SimpleHeap(const Allocator& BlockAllocator = Allocator())
|
||||
: block_allocator_(BlockAllocator), in_use_size_(0), cache_size_(0) {}
|
||||
~SimpleHeap() {
|
||||
trim();
|
||||
// Leak here may be due to the user. Check is for debugging only.
|
||||
// assert(in_use_size_ == 0 && "Leak in SimpleHeap.");
|
||||
}
|
||||
|
||||
SimpleHeap(const SimpleHeap& rhs) = delete;
|
||||
SimpleHeap(SimpleHeap&& rhs) = delete;
|
||||
SimpleHeap& operator=(const SimpleHeap& rhs) = delete;
|
||||
SimpleHeap& operator=(SimpleHeap&& rhs) = delete;
|
||||
|
||||
void* alloc(size_t bytes) {
|
||||
// Find best fit.
|
||||
uintptr_t base;
|
||||
size_t size;
|
||||
// For bytes >= 2MB, the requested mem should be aligned
|
||||
size_t align_bytes = bytes;
|
||||
const int retry = bytes >= GPU_HUGE_PAGE_SIZE ? 1 : 0;
|
||||
size_t align = bytes >= GPU_HUGE_PAGE_SIZE ? GPU_HUGE_PAGE_SIZE : DEFAULT_GPU_PAGE_SIZE;
|
||||
|
||||
for (int i = 0; i <= retry; i++) {
|
||||
auto free_fragment = free_list_.lower_bound(align_bytes);
|
||||
if (free_fragment == free_list_.end()) break;
|
||||
|
||||
uintptr_t addr = free_fragment->second;
|
||||
size = free_fragment->first;
|
||||
|
||||
assert(size >= bytes && "SimpleHeap: map lower_bound failure.");
|
||||
|
||||
// Find the containing block and fragment
|
||||
auto it = block_list_.upper_bound(addr);
|
||||
it--;
|
||||
auto& frag_map = it->second;
|
||||
const auto& fragment = frag_map.find(addr);
|
||||
|
||||
assert(fragment != frag_map.end() && "Inconsistency in SimpleHeap.");
|
||||
assert(size == fragment->second.size && "Inconsistency in SimpleHeap.");
|
||||
|
||||
size_t delta = addr & (align - 1);
|
||||
if (!delta) {
|
||||
// already find aligned address
|
||||
base = addr;
|
||||
free_list_.erase(free_fragment);
|
||||
// Sub-allocate from fragment.
|
||||
fragment->second.size = bytes;
|
||||
setUsed(fragment->second);
|
||||
// Record remaining free space.
|
||||
if (size > bytes) {
|
||||
free_fragment = free_list_.insert(std::make_pair(size - bytes, base + bytes));
|
||||
frag_map[base + bytes] = makeFragment(free_fragment, size - bytes);
|
||||
}
|
||||
} else {
|
||||
// If this is the first request and the requested size is not enough for alignment,
|
||||
// then request for a bigger hole and do trim.
|
||||
if (i == 0 && size < bytes + align - delta) {
|
||||
align_bytes += align;
|
||||
continue;
|
||||
}
|
||||
|
||||
uintptr_t aligned_base = addr + align - delta;
|
||||
base = aligned_base;
|
||||
|
||||
// Erase the old free list
|
||||
free_list_.erase(free_fragment);
|
||||
|
||||
// fragment 1 - free
|
||||
free_fragment = free_list_.insert(std::make_pair(aligned_base - addr, addr));
|
||||
frag_map[addr] = makeFragment(free_fragment, aligned_base - addr);
|
||||
|
||||
//fragment 2 - used
|
||||
frag_map[base] = makeFragment(bytes);
|
||||
|
||||
// fragement 3 - free
|
||||
if (size > aligned_base - addr + bytes) {
|
||||
free_fragment = free_list_.insert(std::make_pair(size - (aligned_base - addr) - bytes, aligned_base + bytes));
|
||||
frag_map[aligned_base + bytes] = makeFragment(free_fragment, size - (aligned_base - addr) - bytes);
|
||||
}
|
||||
}
|
||||
return reinterpret_cast<void*>(base);
|
||||
}
|
||||
|
||||
// No usable fragment, check block cache
|
||||
if (bytes < default_block_size() && !block_cache_.empty()) {
|
||||
const auto& block = block_cache_.back();
|
||||
base = block.base_ptr_;
|
||||
size = block.length_;
|
||||
block_cache_.pop_back();
|
||||
cache_size_ -= size;
|
||||
} else { // Alloc new block - new block may be larger than default.
|
||||
void* ptr = block_allocator_.alloc(bytes, size);
|
||||
base = reinterpret_cast<uintptr_t>(ptr);
|
||||
assert(ptr != nullptr && "Block allocation failed, Allocator is expected to throw.");
|
||||
}
|
||||
|
||||
in_use_size_ += size;
|
||||
assert(size >= bytes && "Alloc exceeds block size.");
|
||||
// Sub alloc and insert free region.
|
||||
if (size > bytes) {
|
||||
auto free_fragment = free_list_.insert(std::make_pair(size - bytes, base + bytes));
|
||||
block_list_[base][base + bytes] = makeFragment(free_fragment, size - bytes);
|
||||
}
|
||||
// Track used region
|
||||
block_list_[base][base] = makeFragment(bytes);
|
||||
|
||||
// Disallow multiple suballocation from large blocks.
|
||||
// Prevents a small allocation from retaining a large block.
|
||||
if (bytes > default_block_size()) {
|
||||
bool err = discardBlock(reinterpret_cast<void*>(base));
|
||||
assert(err && "Large block discard failed.");
|
||||
}
|
||||
|
||||
return reinterpret_cast<void*>(base);
|
||||
}
|
||||
|
||||
bool free(void* ptr) {
|
||||
if (ptr == nullptr) return true;
|
||||
|
||||
uintptr_t base = reinterpret_cast<uintptr_t>(ptr);
|
||||
|
||||
// Find fragment and validate.
|
||||
auto frag_map_it = block_list_.upper_bound(base);
|
||||
if (frag_map_it == block_list_.begin()) return false;
|
||||
frag_map_it--;
|
||||
auto& frag_map = frag_map_it->second;
|
||||
auto fragment = frag_map.find(base);
|
||||
if (fragment == frag_map.end() || isFree(fragment->second)) return false;
|
||||
|
||||
bool discard = fragment->second.discard;
|
||||
|
||||
// Merge lower
|
||||
if (fragment != frag_map.begin()) {
|
||||
auto lower = fragment;
|
||||
lower--;
|
||||
if (isFree(lower->second)) {
|
||||
removeFreeListEntry(lower->second);
|
||||
lower->second.size += fragment->second.size;
|
||||
frag_map.erase(fragment);
|
||||
fragment = lower;
|
||||
}
|
||||
}
|
||||
|
||||
// Merge upper
|
||||
{
|
||||
auto upper = fragment;
|
||||
upper++;
|
||||
if ((upper != frag_map.end()) && isFree(upper->second)) {
|
||||
removeFreeListEntry(upper->second);
|
||||
fragment->second.size += upper->second.size;
|
||||
frag_map.erase(upper);
|
||||
}
|
||||
}
|
||||
|
||||
// Release whole free blocks.
|
||||
if (frag_map.size() == 1) {
|
||||
Block block(fragment->first, fragment->second.size);
|
||||
block_list_.erase(frag_map_it);
|
||||
|
||||
// Discard or add to the block cache.
|
||||
if (discard) {
|
||||
block_allocator_.free(reinterpret_cast<void*>(block.base_ptr_), block.length_);
|
||||
} else {
|
||||
block_cache_.push_back(block);
|
||||
cache_size_ += block.length_;
|
||||
in_use_size_ -= block.length_;
|
||||
}
|
||||
|
||||
balance();
|
||||
|
||||
// Don't publish free space since block was moved to the cache.
|
||||
return true;
|
||||
}
|
||||
|
||||
// Don't report free memory if discarding the fragment.
|
||||
if (discard) return true;
|
||||
|
||||
// Report free fragment
|
||||
const auto& freeEntry =
|
||||
free_list_.insert(std::make_pair(size_t(fragment->second.size), fragment->first));
|
||||
setFree(fragment->second, freeEntry);
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
void balance() {
|
||||
// Release old blocks when over cache limit.
|
||||
while ((block_cache_.size() > 1) && (cache_size_ > in_use_size_ * 2)) {
|
||||
const auto& block = block_cache_.front();
|
||||
block_allocator_.free(reinterpret_cast<void*>(block.base_ptr_), block.length_);
|
||||
cache_size_ -= block.length_;
|
||||
block_cache_.pop_front();
|
||||
}
|
||||
}
|
||||
|
||||
void trim() {
|
||||
for (const auto& block : block_cache_)
|
||||
block_allocator_.free(reinterpret_cast<void*>(block.base_ptr_), block.length_);
|
||||
block_cache_.clear();
|
||||
cache_size_ = 0;
|
||||
}
|
||||
|
||||
size_t cache_size() const { return cache_size_; }
|
||||
|
||||
size_t default_block_size() const { return block_allocator_.block_size(); }
|
||||
|
||||
// Prevent reuse of the block containing ptr. No further fragments will be allocated from the
|
||||
// block and the block will not be added to the block cache when it is free.
|
||||
bool discardBlock(void* ptr) {
|
||||
if (ptr == nullptr) return true;
|
||||
|
||||
uintptr_t base = reinterpret_cast<uintptr_t>(ptr);
|
||||
|
||||
// Find block validate.
|
||||
auto frag_map_it = block_list_.upper_bound(base);
|
||||
if (frag_map_it == block_list_.begin()) return false;
|
||||
frag_map_it--;
|
||||
auto& frag_map = frag_map_it->second;
|
||||
if ((base < frag_map.begin()->first) ||
|
||||
(frag_map.rbegin()->first + frag_map.rbegin()->second.size <= base))
|
||||
return false;
|
||||
|
||||
// Is block already discarded?
|
||||
if (frag_map.begin()->second.discard) return true;
|
||||
|
||||
// Mark all fragments for discard and compute block size. Removes freelist records for all
|
||||
// fragments in the block.
|
||||
size_t size = 0;
|
||||
for (auto& frag : frag_map) {
|
||||
discard(frag.second);
|
||||
size += frag.second.size;
|
||||
}
|
||||
|
||||
// Remove discarded block from in-use tracking and rebalance the block cache.
|
||||
in_use_size_ -= size;
|
||||
balance();
|
||||
|
||||
return true;
|
||||
}
|
||||
};
|
||||
|
||||
} // namespace rocr
|
||||
|
||||
#endif // HSA_RUNTME_CORE_UTIL_SIMPLE_HEAP_H_
|
||||
@@ -0,0 +1,185 @@
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
//
|
||||
// The University of Illinois/NCSA
|
||||
// Open Source License (NCSA)
|
||||
//
|
||||
// Copyright (c) 2014-2020, Advanced Micro Devices, Inc. All rights reserved.
|
||||
//
|
||||
// Developed by:
|
||||
//
|
||||
// AMD Research and AMD HSA Software Development
|
||||
//
|
||||
// Advanced Micro Devices, Inc.
|
||||
//
|
||||
// www.amd.com
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
// of this software and associated documentation files (the "Software"), to
|
||||
// deal with the Software without restriction, including without limitation
|
||||
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
// and/or sell copies of the Software, and to permit persons to whom the
|
||||
// Software is furnished to do so, subject to the following conditions:
|
||||
//
|
||||
// - Redistributions of source code must retain the above copyright notice,
|
||||
// this list of conditions and the following disclaimers.
|
||||
// - Redistributions in binary form must reproduce the above copyright
|
||||
// notice, this list of conditions and the following disclaimers in
|
||||
// the documentation and/or other materials provided with the distribution.
|
||||
// - Neither the names of Advanced Micro Devices, Inc,
|
||||
// nor the names of its contributors may be used to endorse or promote
|
||||
// products derived from this Software without specific prior written
|
||||
// permission.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
|
||||
// THE CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
|
||||
// OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
|
||||
// ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
||||
// DEALINGS WITH THE SOFTWARE.
|
||||
//
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
#include "small_heap.h"
|
||||
|
||||
namespace rocr {
|
||||
|
||||
// Inserts node into freelist after place.
|
||||
// Assumes node will not be an end of the list (list has guard nodes).
|
||||
void SmallHeap::insertafter(SmallHeap::iterator_t place, SmallHeap::iterator_t node) {
|
||||
assert(place->first < node->first && "Order violation");
|
||||
assert(isfree(place->second) && "Freelist operation error.");
|
||||
iterator_t next = place->second.next;
|
||||
node->second.next = next;
|
||||
node->second.prior = place;
|
||||
place->second.next = node;
|
||||
next->second.prior = node;
|
||||
}
|
||||
|
||||
// Removes node from freelist.
|
||||
// Assumes node will not be an end of the list (list has guard nodes).
|
||||
void SmallHeap::remove(SmallHeap::iterator_t node) {
|
||||
assert(isfree(node->second) && "Freelist operation error.");
|
||||
node->second.prior->second.next = node->second.next;
|
||||
node->second.next->second.prior = node->second.prior;
|
||||
setused(node->second);
|
||||
}
|
||||
|
||||
// Returns high if merge failed or the merged node.
|
||||
SmallHeap::memory_t::iterator SmallHeap::merge(SmallHeap::memory_t::iterator low,
|
||||
SmallHeap::memory_t::iterator high) {
|
||||
assert(isfree(low->second) && "Merge with allocated block");
|
||||
assert(isfree(high->second) && "Merge with allocated block");
|
||||
|
||||
if ((char*)low->first + low->second.len != (char*)high->first) return high;
|
||||
|
||||
assert(!islastfree(high->second) && "Illegal merge.");
|
||||
|
||||
low->second.len += high->second.len;
|
||||
low->second.next = high->second.next;
|
||||
high->second.next->second.prior = low;
|
||||
|
||||
memory.erase(high);
|
||||
return low;
|
||||
}
|
||||
|
||||
void SmallHeap::free(void* ptr) {
|
||||
if (ptr == nullptr) return;
|
||||
|
||||
auto iterator = memory.find(ptr);
|
||||
|
||||
// Check for illegal free
|
||||
if (iterator == memory.end()) {
|
||||
assert(false && "Illegal free.");
|
||||
return;
|
||||
}
|
||||
|
||||
// Return memory to total and link node into free list
|
||||
total_free += iterator->second.len;
|
||||
|
||||
// Could also traverse the free list which might be faster in some cases.
|
||||
auto before = iterator;
|
||||
before--;
|
||||
while (!isfree(before->second)) before--;
|
||||
assert(before->second.next->first > iterator->first && "Inconsistency in small heap.");
|
||||
insertafter(before, iterator);
|
||||
|
||||
// Attempt compaction
|
||||
iterator = merge(before, iterator);
|
||||
merge(iterator, iterator->second.next);
|
||||
|
||||
// Update lowHighBondary
|
||||
high.erase(ptr);
|
||||
}
|
||||
|
||||
void* SmallHeap::alloc(size_t bytes) {
|
||||
// Is enough memory available?
|
||||
if ((bytes > total_free) || (bytes == 0)) return nullptr;
|
||||
|
||||
iterator_t current;
|
||||
|
||||
// Walk the free list and allocate at first fitting location
|
||||
current = firstfree();
|
||||
while (!islastfree(current->second)) {
|
||||
if (bytes <= current->second.len) {
|
||||
// Decrement from total
|
||||
total_free -= bytes;
|
||||
|
||||
// Split node
|
||||
if (bytes != current->second.len) {
|
||||
void* remaining = (char*)current->first + bytes;
|
||||
Node& node = memory[remaining];
|
||||
node.len = current->second.len - bytes;
|
||||
current->second.len = bytes;
|
||||
insertafter(current, memory.find(remaining));
|
||||
}
|
||||
|
||||
remove(current);
|
||||
return current->first;
|
||||
}
|
||||
current = current->second.next;
|
||||
}
|
||||
assert(current->second.len == 0 && "Freelist corruption.");
|
||||
|
||||
// Can't service the request due to fragmentation
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
void* SmallHeap::alloc_high(size_t bytes) {
|
||||
// Is enough memory available?
|
||||
if ((bytes > total_free) || (bytes == 0)) return nullptr;
|
||||
|
||||
iterator_t current;
|
||||
|
||||
// Walk the free list and allocate at first fitting location
|
||||
current = lastfree();
|
||||
while (!isfirstfree(current->second)) {
|
||||
if (bytes <= current->second.len) {
|
||||
// Decrement from total
|
||||
total_free -= bytes;
|
||||
|
||||
void* alloc;
|
||||
// Split node
|
||||
if (bytes != current->second.len) {
|
||||
alloc = (char*)current->first + current->second.len - bytes;
|
||||
current->second.len -= bytes;
|
||||
Node& node = memory[alloc];
|
||||
node.len = bytes;
|
||||
setused(node);
|
||||
} else {
|
||||
alloc = current->first;
|
||||
remove(current);
|
||||
}
|
||||
|
||||
high.insert(alloc);
|
||||
return alloc;
|
||||
}
|
||||
current = current->second.prior;
|
||||
}
|
||||
assert(current->second.len == 0 && "Freelist corruption.");
|
||||
|
||||
// Can't service the request due to fragmentation
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
} // namespace rocr
|
||||
@@ -0,0 +1,131 @@
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
//
|
||||
// The University of Illinois/NCSA
|
||||
// Open Source License (NCSA)
|
||||
//
|
||||
// Copyright (c) 2014-2020, Advanced Micro Devices, Inc. All rights reserved.
|
||||
//
|
||||
// Developed by:
|
||||
//
|
||||
// AMD Research and AMD HSA Software Development
|
||||
//
|
||||
// Advanced Micro Devices, Inc.
|
||||
//
|
||||
// www.amd.com
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
// of this software and associated documentation files (the "Software"), to
|
||||
// deal with the Software without restriction, including without limitation
|
||||
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
// and/or sell copies of the Software, and to permit persons to whom the
|
||||
// Software is furnished to do so, subject to the following conditions:
|
||||
//
|
||||
// - Redistributions of source code must retain the above copyright notice,
|
||||
// this list of conditions and the following disclaimers.
|
||||
// - Redistributions in binary form must reproduce the above copyright
|
||||
// notice, this list of conditions and the following disclaimers in
|
||||
// the documentation and/or other materials provided with the distribution.
|
||||
// - Neither the names of Advanced Micro Devices, Inc,
|
||||
// nor the names of its contributors may be used to endorse or promote
|
||||
// products derived from this Software without specific prior written
|
||||
// permission.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
|
||||
// THE CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
|
||||
// OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
|
||||
// ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
||||
// DEALINGS WITH THE SOFTWARE.
|
||||
//
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
// A simple first fit memory allocator with eager compaction. For use with few
|
||||
// items (where list iteration is faster than trees).
|
||||
// Not thread safe!
|
||||
|
||||
#ifndef HSA_RUNTME_CORE_UTIL_SMALL_HEAP_H_
|
||||
#define HSA_RUNTME_CORE_UTIL_SMALL_HEAP_H_
|
||||
|
||||
#include <map>
|
||||
#include <set>
|
||||
|
||||
#include "utils.h"
|
||||
|
||||
namespace rocr {
|
||||
|
||||
class SmallHeap {
|
||||
private:
|
||||
struct Node;
|
||||
typedef std::map<void*, Node> memory_t;
|
||||
typedef memory_t::iterator iterator_t;
|
||||
|
||||
struct Node {
|
||||
size_t len;
|
||||
iterator_t next;
|
||||
iterator_t prior;
|
||||
};
|
||||
|
||||
SmallHeap(const SmallHeap& rhs) = delete;
|
||||
SmallHeap& operator=(const SmallHeap& rhs) = delete;
|
||||
|
||||
void* const pool;
|
||||
const size_t length;
|
||||
|
||||
size_t total_free;
|
||||
memory_t memory;
|
||||
std::set<void*> high;
|
||||
|
||||
__forceinline bool isfree(const Node& node) const { return node.next != memory.begin(); }
|
||||
__forceinline bool islastfree(const Node& node) const { return node.next == memory.end(); }
|
||||
__forceinline bool isfirstfree(const Node& node) const { return node.prior == memory.end(); }
|
||||
__forceinline void setlastfree(Node& node) { node.next = memory.end(); }
|
||||
__forceinline void setfirstfree(Node& node) { node.prior = memory.end(); }
|
||||
__forceinline void setused(Node& node) { node.next = memory.begin(); }
|
||||
|
||||
__forceinline iterator_t firstfree() { return memory.begin()->second.next; }
|
||||
__forceinline iterator_t lastfree() { return memory.rbegin()->second.prior; }
|
||||
void insertafter(iterator_t place, iterator_t node);
|
||||
void remove(iterator_t node);
|
||||
iterator_t merge(iterator_t low, iterator_t high);
|
||||
|
||||
public:
|
||||
SmallHeap() : pool(nullptr), length(0), total_free(0) {}
|
||||
SmallHeap(void* base, size_t length)
|
||||
: pool(base), length(length), total_free(length) {
|
||||
assert(pool != nullptr && "Invalid base address.");
|
||||
assert(pool != (void*)0xFFFFFFFFFFFFFFFFull && "Invalid base address.");
|
||||
assert((char*)pool + length != (char*)0xFFFFFFFFFFFFFFFFull && "Invalid pool bounds.");
|
||||
|
||||
Node& start = memory[0];
|
||||
Node& node = memory[pool];
|
||||
Node& end = memory[(void*)0xFFFFFFFFFFFFFFFFull];
|
||||
|
||||
start.len = 0;
|
||||
start.next = memory.find(pool);
|
||||
setfirstfree(start);
|
||||
|
||||
node.len = length;
|
||||
node.prior = memory.begin();
|
||||
node.next = --memory.end();
|
||||
|
||||
end.len = 0;
|
||||
end.prior = start.next;
|
||||
setlastfree(end);
|
||||
|
||||
high.insert((void*)0xFFFFFFFFFFFFFFFFull);
|
||||
}
|
||||
|
||||
void* alloc(size_t bytes);
|
||||
void* alloc_high(size_t bytes);
|
||||
void free(void* ptr);
|
||||
|
||||
void* base() const { return pool; }
|
||||
size_t size() const { return length; }
|
||||
size_t remaining() const { return total_free; }
|
||||
void* high_split() const { return *high.begin(); }
|
||||
};
|
||||
|
||||
} // namespace rocr
|
||||
|
||||
#endif
|
||||
@@ -0,0 +1,111 @@
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
//
|
||||
// The University of Illinois/NCSA
|
||||
// Open Source License (NCSA)
|
||||
//
|
||||
// Copyright (c) 2014-2020, Advanced Micro Devices, Inc. All rights reserved.
|
||||
//
|
||||
// Developed by:
|
||||
//
|
||||
// AMD Research and AMD HSA Software Development
|
||||
//
|
||||
// Advanced Micro Devices, Inc.
|
||||
//
|
||||
// www.amd.com
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
// of this software and associated documentation files (the "Software"), to
|
||||
// deal with the Software without restriction, including without limitation
|
||||
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
// and/or sell copies of the Software, and to permit persons to whom the
|
||||
// Software is furnished to do so, subject to the following conditions:
|
||||
//
|
||||
// - Redistributions of source code must retain the above copyright notice,
|
||||
// this list of conditions and the following disclaimers.
|
||||
// - Redistributions in binary form must reproduce the above copyright
|
||||
// notice, this list of conditions and the following disclaimers in
|
||||
// the documentation and/or other materials provided with the distribution.
|
||||
// - Neither the names of Advanced Micro Devices, Inc,
|
||||
// nor the names of its contributors may be used to endorse or promote
|
||||
// products derived from this Software without specific prior written
|
||||
// permission.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
|
||||
// THE CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
|
||||
// OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
|
||||
// ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
||||
// DEALINGS WITH THE SOFTWARE.
|
||||
//
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
#include "core/util/timer.h"
|
||||
|
||||
namespace rocr {
|
||||
namespace timer {
|
||||
|
||||
accurate_clock::init::init() {
|
||||
freq = os::AccurateClockFrequency();
|
||||
accurate_clock::period_ns = 1e9 / double(freq);
|
||||
}
|
||||
|
||||
// Calibrates the fast clock using the accurate clock.
|
||||
fast_clock::init::init() {
|
||||
typedef accurate_clock clock;
|
||||
clock::duration delay(std::chrono::milliseconds(1));
|
||||
|
||||
// calibrate clock
|
||||
fast_clock::raw_rep min = 0;
|
||||
clock::duration elapsed;
|
||||
|
||||
do {
|
||||
elapsed = clock::duration::max();
|
||||
|
||||
for (int t = 0; t < 10; t++) {
|
||||
fast_clock::raw_rep r1, r2;
|
||||
clock::time_point t0, t1, t2, t3;
|
||||
|
||||
t0 = clock::now();
|
||||
std::atomic_signal_fence(std::memory_order_acq_rel);
|
||||
r1 = fast_clock::raw_now();
|
||||
std::atomic_signal_fence(std::memory_order_acq_rel);
|
||||
t1 = clock::now();
|
||||
std::atomic_signal_fence(std::memory_order_acq_rel);
|
||||
|
||||
do {
|
||||
t2 = clock::now();
|
||||
} while (t2 - t1 < delay);
|
||||
|
||||
std::atomic_signal_fence(std::memory_order_acq_rel);
|
||||
r2 = fast_clock::raw_now();
|
||||
std::atomic_signal_fence(std::memory_order_acq_rel);
|
||||
t3 = clock::now();
|
||||
|
||||
// If elapsed time is shorter than last recorded time and both the start
|
||||
// and end times are confirmed correlated then record the clock readings.
|
||||
// This protects against inaccuracy due to thread switching
|
||||
if ((t3 - t1 < elapsed) && ((t1 - t0) * 10 < (t2 - t1)) &&
|
||||
((t3 - t2) * 10 < (t2 - t1))) {
|
||||
elapsed = t3 - t1;
|
||||
min = r2 - r1;
|
||||
}
|
||||
}
|
||||
delay += delay;
|
||||
} while (min < 1000);
|
||||
|
||||
fast_clock::freq = double(min) / duration_in_seconds(elapsed);
|
||||
fast_clock::period_ps = 1e12 / fast_clock::freq;
|
||||
// printf("Timer setup took %f ms\n", duration_in_seconds(elapsed)*1000.0f);
|
||||
// printf("Fast clock frequency: %f MHz\n", double(fast_clock::freq)/1e6);
|
||||
}
|
||||
|
||||
double accurate_clock::period_ns;
|
||||
accurate_clock::raw_frequency accurate_clock::freq;
|
||||
accurate_clock::init accurate_clock::accurate_clock_init;
|
||||
|
||||
double fast_clock::period_ps;
|
||||
fast_clock::raw_frequency fast_clock::freq;
|
||||
fast_clock::init fast_clock::fast_clock_init;
|
||||
} // namespace timer
|
||||
} // namespace rocr
|
||||
@@ -0,0 +1,173 @@
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
//
|
||||
// The University of Illinois/NCSA
|
||||
// Open Source License (NCSA)
|
||||
//
|
||||
// Copyright (c) 2014-2020, Advanced Micro Devices, Inc. All rights reserved.
|
||||
//
|
||||
// Developed by:
|
||||
//
|
||||
// AMD Research and AMD HSA Software Development
|
||||
//
|
||||
// Advanced Micro Devices, Inc.
|
||||
//
|
||||
// www.amd.com
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
// of this software and associated documentation files (the "Software"), to
|
||||
// deal with the Software without restriction, including without limitation
|
||||
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
// and/or sell copies of the Software, and to permit persons to whom the
|
||||
// Software is furnished to do so, subject to the following conditions:
|
||||
//
|
||||
// - Redistributions of source code must retain the above copyright notice,
|
||||
// this list of conditions and the following disclaimers.
|
||||
// - Redistributions in binary form must reproduce the above copyright
|
||||
// notice, this list of conditions and the following disclaimers in
|
||||
// the documentation and/or other materials provided with the distribution.
|
||||
// - Neither the names of Advanced Micro Devices, Inc,
|
||||
// nor the names of its contributors may be used to endorse or promote
|
||||
// products derived from this Software without specific prior written
|
||||
// permission.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
|
||||
// THE CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
|
||||
// OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
|
||||
// ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
||||
// DEALINGS WITH THE SOFTWARE.
|
||||
//
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
#ifndef HSA_RUNTIME_CORE_UTIL_TIMER_H_
|
||||
#define HSA_RUNTIME_CORE_UTIL_TIMER_H_
|
||||
|
||||
#include "core/util/utils.h"
|
||||
#include "core/util/os.h"
|
||||
#include <chrono>
|
||||
#include <time.h>
|
||||
#include <type_traits>
|
||||
|
||||
namespace rocr {
|
||||
namespace timer {
|
||||
|
||||
// Needed to patch around a mixed arithmetic bug in MSVC's duration_cast as of
|
||||
// VS 2013.
|
||||
template <bool isFloat, bool isSigned>
|
||||
struct wide_type {
|
||||
typedef double type;
|
||||
};
|
||||
template <>
|
||||
struct wide_type<false, false> {
|
||||
typedef uintmax_t type;
|
||||
};
|
||||
template <>
|
||||
struct wide_type<false, true> {
|
||||
typedef intmax_t type;
|
||||
};
|
||||
|
||||
template <typename To, typename Rep, typename Period>
|
||||
static __forceinline To
|
||||
duration_cast(const std::chrono::duration<Rep, Period>& d) {
|
||||
typedef typename wide_type<std::is_floating_point<Rep>::value,
|
||||
std::is_signed<Rep>::value>::type wide;
|
||||
typedef std::chrono::duration<wide, typename To::period> unit_convert_t;
|
||||
|
||||
unit_convert_t temp = std::chrono::duration_cast<unit_convert_t>(d);
|
||||
return To(static_cast<typename To::rep>(temp.count()));
|
||||
}
|
||||
// End patch
|
||||
|
||||
template <typename Rep, typename Period>
|
||||
static __forceinline double duration_in_seconds(
|
||||
std::chrono::duration<Rep, Period> delta) {
|
||||
typedef std::chrono::duration<double, std::ratio<1, 1>> seconds;
|
||||
return seconds(delta).count();
|
||||
}
|
||||
|
||||
template <typename rep>
|
||||
static __forceinline rep duration_from_seconds(double delta) {
|
||||
typedef std::chrono::duration<double, std::ratio<1, 1>> seconds;
|
||||
return std::chrono::duration_cast<rep>(seconds(delta));
|
||||
}
|
||||
|
||||
// Provices a C++11 standard clock interface to the os::AccurateClock functions
|
||||
class accurate_clock {
|
||||
public:
|
||||
typedef double rep;
|
||||
typedef std::nano period;
|
||||
typedef std::chrono::duration<rep, period> duration;
|
||||
typedef std::chrono::time_point<accurate_clock> time_point;
|
||||
|
||||
static const bool is_steady = true;
|
||||
|
||||
static __forceinline time_point now() {
|
||||
return time_point(duration(raw_now() * period_ns));
|
||||
}
|
||||
|
||||
// These two extra APIs and types let us use clocks without conversion to the
|
||||
// arbitrary period unit
|
||||
typedef uint64_t raw_rep;
|
||||
typedef uint64_t raw_frequency;
|
||||
|
||||
static __forceinline raw_rep raw_now() { return os::ReadAccurateClock(); }
|
||||
static __forceinline raw_frequency raw_freq() { return freq; }
|
||||
|
||||
private:
|
||||
static double period_ns;
|
||||
static raw_frequency freq;
|
||||
|
||||
class init {
|
||||
public:
|
||||
init();
|
||||
};
|
||||
static init accurate_clock_init;
|
||||
};
|
||||
|
||||
// Provices a C++11 standard clock interface to the lowest latency approximate
|
||||
// clock
|
||||
class fast_clock {
|
||||
public:
|
||||
typedef double rep;
|
||||
typedef std::pico period;
|
||||
typedef std::chrono::duration<rep, period> duration;
|
||||
typedef std::chrono::time_point<fast_clock> time_point;
|
||||
|
||||
static const bool is_steady = true;
|
||||
|
||||
static __forceinline time_point now() {
|
||||
return time_point(duration(raw_now() * period_ps));
|
||||
}
|
||||
|
||||
// These two extra APIs and types let us use clocks without conversion to the
|
||||
// arbitrary period unit
|
||||
typedef uint64_t raw_rep;
|
||||
typedef double raw_frequency;
|
||||
|
||||
#if defined(__x86_64__) || defined(_M_X64)
|
||||
static __forceinline raw_rep raw_now() { return __rdtsc(); }
|
||||
static __forceinline raw_frequency raw_freq() { return freq; }
|
||||
#else
|
||||
static __forceinline raw_rep raw_now() {
|
||||
struct timespec ts;
|
||||
clock_gettime(CLOCK_MONOTONIC_RAW, &ts);
|
||||
return (raw_rep(ts.tv_sec) * 1000000000 + raw_rep(ts.tv_nsec));
|
||||
}
|
||||
static __forceinline raw_frequency raw_freq() { return 1.e-9; }
|
||||
#endif
|
||||
|
||||
private:
|
||||
static double period_ps;
|
||||
static raw_frequency freq;
|
||||
|
||||
class init {
|
||||
public:
|
||||
init();
|
||||
};
|
||||
static init fast_clock_init;
|
||||
};
|
||||
} // namespace timer
|
||||
} // namespace rocr
|
||||
|
||||
#endif
|
||||
@@ -0,0 +1,424 @@
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
//
|
||||
// The University of Illinois/NCSA
|
||||
// Open Source License (NCSA)
|
||||
//
|
||||
// Copyright (c) 2014-2024, Advanced Micro Devices, Inc. All rights reserved.
|
||||
//
|
||||
// Developed by:
|
||||
//
|
||||
// AMD Research and AMD HSA Software Development
|
||||
//
|
||||
// Advanced Micro Devices, Inc.
|
||||
//
|
||||
// www.amd.com
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
// of this software and associated documentation files (the "Software"), to
|
||||
// deal with the Software without restriction, including without limitation
|
||||
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
// and/or sell copies of the Software, and to permit persons to whom the
|
||||
// Software is furnished to do so, subject to the following conditions:
|
||||
//
|
||||
// - Redistributions of source code must retain the above copyright notice,
|
||||
// this list of conditions and the following disclaimers.
|
||||
// - Redistributions in binary form must reproduce the above copyright
|
||||
// notice, this list of conditions and the following disclaimers in
|
||||
// the documentation and/or other materials provided with the distribution.
|
||||
// - Neither the names of Advanced Micro Devices, Inc,
|
||||
// nor the names of its contributors may be used to endorse or promote
|
||||
// products derived from this Software without specific prior written
|
||||
// permission.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
|
||||
// THE CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
|
||||
// OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
|
||||
// ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
||||
// DEALINGS WITH THE SOFTWARE.
|
||||
//
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
// Generally useful utility functions
|
||||
|
||||
#ifndef HSA_RUNTIME_CORE_UTIL_UTILS_H_
|
||||
#define HSA_RUNTIME_CORE_UTIL_UTILS_H_
|
||||
|
||||
#include "stdint.h"
|
||||
#include "stddef.h"
|
||||
#include "stdlib.h"
|
||||
#include "stdarg.h"
|
||||
#include "unistd.h"
|
||||
#include <assert.h>
|
||||
#include <iostream>
|
||||
#include <string>
|
||||
#include <algorithm>
|
||||
#include <sstream>
|
||||
#include <thread>
|
||||
|
||||
namespace rocr {
|
||||
extern FILE* log_file;
|
||||
extern uint8_t log_flags[8];
|
||||
|
||||
typedef unsigned int uint;
|
||||
typedef uint64_t uint64;
|
||||
|
||||
#if defined(__GNUC__)
|
||||
#if defined(__i386__) || defined(__x86_64__)
|
||||
#include <x86intrin.h>
|
||||
#endif
|
||||
|
||||
// 2MB huge page size
|
||||
#define GPU_HUGE_PAGE_SIZE (2 << 20)
|
||||
|
||||
// 4KB page size
|
||||
#define DEFAULT_GPU_PAGE_SIZE (1 << 12)
|
||||
|
||||
#define __forceinline __inline__ __attribute__((always_inline))
|
||||
#define __declspec(x) __attribute__((x))
|
||||
#undef __stdcall
|
||||
#define __stdcall // __attribute__((__stdcall__))
|
||||
#define __ALIGNED__(x) __attribute__((aligned(x)))
|
||||
|
||||
void log_printf(const char* file, int line, const char* format, ...);
|
||||
|
||||
static __forceinline void* _aligned_malloc(size_t size, size_t alignment) {
|
||||
#ifdef _ISOC11_SOURCE
|
||||
return aligned_alloc(alignment, size);
|
||||
#else
|
||||
void *mem = NULL;
|
||||
if (0 != posix_memalign(&mem, alignment, size)) return NULL;
|
||||
return mem;
|
||||
#endif
|
||||
}
|
||||
static __forceinline void _aligned_free(void* ptr) { return free(ptr); }
|
||||
#elif defined(_MSC_VER) && (defined(_M_IX86) || defined(_M_X64))
|
||||
#include "intrin.h"
|
||||
#define __ALIGNED__(x) __declspec(align(x))
|
||||
#if (_MSC_VER < 1800) // < VS 2013
|
||||
static __forceinline unsigned long long int strtoull(const char* str,
|
||||
char** endptr, int base) {
|
||||
return static_cast<unsigned long long>(_strtoui64(str, endptr, base));
|
||||
}
|
||||
#endif
|
||||
#if (_MSC_VER < 1900) // < VS 2015
|
||||
#define thread_local __declspec(thread)
|
||||
#endif
|
||||
#else
|
||||
#error "Compiler and/or processor not identified."
|
||||
#endif
|
||||
|
||||
#define STRING2(x) #x
|
||||
#define STRING(x) STRING2(x)
|
||||
|
||||
#define PASTE2(x, y) x##y
|
||||
#define PASTE(x, y) PASTE2(x, y)
|
||||
|
||||
#ifdef NDEBUG
|
||||
#define debug_warning_n(exp, limit) \
|
||||
do { \
|
||||
} while (false)
|
||||
#else
|
||||
#define debug_warning_n(exp, limit) \
|
||||
do { \
|
||||
static std::atomic<int> count(0); \
|
||||
if (!(exp) && (limit == 0 || count < limit)) { \
|
||||
fprintf(stderr, "Warning: " STRING(exp) " in %s, " __FILE__ ":" STRING(__LINE__) "\n", \
|
||||
__PRETTY_FUNCTION__); \
|
||||
count++; \
|
||||
} \
|
||||
} while (false)
|
||||
#endif
|
||||
#define debug_warning(exp) debug_warning_n((exp), 0)
|
||||
|
||||
#ifdef NDEBUG
|
||||
#define debug_print(fmt, ...) \
|
||||
do { \
|
||||
} while (false)
|
||||
#else
|
||||
#define debug_print(fmt, ...) \
|
||||
do { \
|
||||
fprintf(stderr, fmt, ##__VA_ARGS__); \
|
||||
} while (false)
|
||||
#endif
|
||||
|
||||
#ifdef NDEBUG
|
||||
#define ifdebug if (false)
|
||||
#else
|
||||
#define ifdebug if (true)
|
||||
#endif
|
||||
|
||||
#define __FILENAME__ (strrchr(__FILE__, '/') ? strrchr(__FILE__, '/') + 1 : __FILE__)
|
||||
|
||||
#define LogPrint(flag, format, ...) \
|
||||
do { \
|
||||
if (hsa_flag_isset64(log_flags, flag)) \
|
||||
rocr::log_printf(__FILENAME__, __LINE__, format, ##__VA_ARGS__); \
|
||||
} while (false);
|
||||
|
||||
|
||||
// A macro to disallow the copy and move constructor and operator= functions
|
||||
#define DISALLOW_COPY_AND_ASSIGN(TypeName) \
|
||||
TypeName(const TypeName&) = delete; \
|
||||
TypeName(TypeName&&) = delete; \
|
||||
void operator=(const TypeName&) = delete; \
|
||||
void operator=(TypeName&&) = delete;
|
||||
|
||||
template <typename lambda>
|
||||
class ScopeGuard {
|
||||
public:
|
||||
explicit __forceinline ScopeGuard(const lambda& release)
|
||||
: release_(release), dismiss_(false) {}
|
||||
|
||||
ScopeGuard(ScopeGuard& rhs) { *this = rhs; }
|
||||
|
||||
__forceinline ~ScopeGuard() {
|
||||
if (!dismiss_) release_();
|
||||
}
|
||||
__forceinline ScopeGuard& operator=(ScopeGuard& rhs) {
|
||||
dismiss_ = rhs.dismiss_;
|
||||
release_ = rhs.release_;
|
||||
rhs.dismiss_ = true;
|
||||
return *this;
|
||||
}
|
||||
__forceinline void Dismiss() { dismiss_ = true; }
|
||||
|
||||
private:
|
||||
lambda release_;
|
||||
bool dismiss_;
|
||||
};
|
||||
|
||||
template <typename lambda>
|
||||
static __forceinline ScopeGuard<lambda> MakeScopeGuard(lambda rel) {
|
||||
return ScopeGuard<lambda>(rel);
|
||||
}
|
||||
|
||||
#define MAKE_SCOPE_GUARD_HELPER(lname, sname, ...) \
|
||||
auto lname = __VA_ARGS__; \
|
||||
ScopeGuard<decltype(lname)> sname(lname);
|
||||
#define MAKE_SCOPE_GUARD(...) \
|
||||
MAKE_SCOPE_GUARD_HELPER(PASTE(scopeGuardLambda, __COUNTER__), \
|
||||
PASTE(scopeGuard, __COUNTER__), __VA_ARGS__)
|
||||
#define MAKE_NAMED_SCOPE_GUARD(name, ...) \
|
||||
MAKE_SCOPE_GUARD_HELPER(PASTE(scopeGuardLambda, __COUNTER__), name, \
|
||||
__VA_ARGS__)
|
||||
|
||||
/// @brief: Finds out the min one of two inputs, input must support ">"
|
||||
/// operator.
|
||||
/// @param: a(Input), a reference to type T.
|
||||
/// @param: b(Input), a reference to type T.
|
||||
/// @return: T.
|
||||
template <class T>
|
||||
static __forceinline T Min(const T& a, const T& b) {
|
||||
return (a > b) ? b : a;
|
||||
}
|
||||
|
||||
template <class T, class... Arg>
|
||||
static __forceinline T Min(const T& a, const T& b, Arg... args) {
|
||||
return Min(a, Min(b, args...));
|
||||
}
|
||||
|
||||
/// @brief: Find out the max one of two inputs, input must support ">" operator.
|
||||
/// @param: a(Input), a reference to type T.
|
||||
/// @param: b(Input), a reference to type T.
|
||||
/// @return: T.
|
||||
template <class T>
|
||||
static __forceinline T Max(const T& a, const T& b) {
|
||||
return (b > a) ? b : a;
|
||||
}
|
||||
|
||||
template <class T, class... Arg>
|
||||
static __forceinline T Max(const T& a, const T& b, Arg... args) {
|
||||
return Max(a, Max(b, args...));
|
||||
}
|
||||
|
||||
/// @brief: Free the memory space which is newed previously.
|
||||
/// @param: ptr(Input), a pointer to memory space. Can't be NULL.
|
||||
/// @return: void.
|
||||
struct DeleteObject {
|
||||
template <typename T>
|
||||
void operator()(const T* ptr) const {
|
||||
delete ptr;
|
||||
}
|
||||
};
|
||||
|
||||
/// @brief: Checks if a value is power of two, if it is, return true. Be careful
|
||||
/// when passing 0.
|
||||
/// @param: val(Input), the data to be checked.
|
||||
/// @return: bool.
|
||||
template <typename T>
|
||||
static __forceinline bool IsPowerOfTwo(T val) {
|
||||
return (val & (val - 1)) == 0;
|
||||
}
|
||||
|
||||
/// @brief: Calculates the floor value aligned based on parameter of alignment.
|
||||
/// If value is at the boundary of alignment, it is unchanged.
|
||||
/// @param: value(Input), value to be calculated.
|
||||
/// @param: alignment(Input), alignment value.
|
||||
/// @return: T.
|
||||
template <typename T>
|
||||
static __forceinline T AlignDown(T value, size_t alignment) {
|
||||
return (T)((value / alignment) * alignment);
|
||||
}
|
||||
|
||||
/// @brief: Same as previous one, but first parameter becomes pointer, for more
|
||||
/// info, see the previous desciption.
|
||||
/// @param: value(Input), pointer to type T.
|
||||
/// @param: alignment(Input), alignment value.
|
||||
/// @return: T*, pointer to type T.
|
||||
template <typename T>
|
||||
static __forceinline T* AlignDown(T* value, size_t alignment) {
|
||||
return (T*)AlignDown((intptr_t)value, alignment);
|
||||
}
|
||||
|
||||
/// @brief: Calculates the ceiling value aligned based on parameter of
|
||||
/// alignment.
|
||||
/// If value is at the boundary of alignment, it is unchanged.
|
||||
/// @param: value(Input), value to be calculated.
|
||||
/// @param: alignment(Input), alignment value.
|
||||
/// @param: T.
|
||||
template <typename T>
|
||||
static __forceinline T AlignUp(T value, size_t alignment) {
|
||||
return AlignDown((T)(value + alignment - 1), alignment);
|
||||
}
|
||||
|
||||
/// @brief: Same as previous one, but first parameter becomes pointer, for more
|
||||
/// info, see the previous desciption.
|
||||
/// @param: value(Input), pointer to type T.
|
||||
/// @param: alignment(Input), alignment value.
|
||||
/// @return: T*, pointer to type T.
|
||||
template <typename T>
|
||||
static __forceinline T* AlignUp(T* value, size_t alignment) {
|
||||
return (T*)AlignDown((intptr_t)((uint8_t*)value + alignment - 1), alignment);
|
||||
}
|
||||
|
||||
/// @brief: Checks if the input value is at the boundary of alignment, if it is,
|
||||
/// @return true.
|
||||
/// @param: value(Input), value to be checked.
|
||||
/// @param: alignment(Input), alignment value.
|
||||
/// @return: bool.
|
||||
template <typename T>
|
||||
static __forceinline bool IsMultipleOf(T value, size_t alignment) {
|
||||
return (AlignUp(value, alignment) == value);
|
||||
}
|
||||
|
||||
/// @brief: Same as previous one, but first parameter becomes pointer, for more
|
||||
/// info, see the previous desciption.
|
||||
/// @param: value(Input), pointer to type T.
|
||||
/// @param: alignment(Input), alignment value.
|
||||
/// @return: bool.
|
||||
template <typename T>
|
||||
static __forceinline bool IsMultipleOf(T* value, size_t alignment) {
|
||||
return (AlignUp(value, alignment) == value);
|
||||
}
|
||||
|
||||
static __forceinline uint32_t NextPow2(uint32_t value) {
|
||||
if (value == 0) return 1;
|
||||
uint32_t v = value - 1;
|
||||
v |= v >> 1;
|
||||
v |= v >> 2;
|
||||
v |= v >> 4;
|
||||
v |= v >> 8;
|
||||
v |= v >> 16;
|
||||
return v + 1;
|
||||
}
|
||||
|
||||
static __forceinline uint64_t NextPow2(uint64_t value) {
|
||||
if (value == 0) return 1;
|
||||
uint64_t v = value - 1;
|
||||
v |= v >> 1;
|
||||
v |= v >> 2;
|
||||
v |= v >> 4;
|
||||
v |= v >> 8;
|
||||
v |= v >> 16;
|
||||
v |= v >> 32;
|
||||
return v + 1;
|
||||
}
|
||||
|
||||
static __forceinline bool strIsEmpty(const char* str) noexcept { return str[0] == '\0'; }
|
||||
|
||||
static __forceinline std::string& ltrim(std::string& s) {
|
||||
auto it = std::find_if(s.begin(), s.end(),
|
||||
[](char c) { return !std::isspace<char>(c, std::locale::classic()); });
|
||||
s.erase(s.begin(), it);
|
||||
return s;
|
||||
}
|
||||
|
||||
static __forceinline std::string& rtrim(std::string& s) {
|
||||
auto it = std::find_if(s.rbegin(), s.rend(),
|
||||
[](char c) { return !std::isspace<char>(c, std::locale::classic()); });
|
||||
s.erase(it.base(), s.end());
|
||||
return s;
|
||||
}
|
||||
|
||||
static __forceinline std::string& trim(std::string& s) { return ltrim(rtrim(s)); }
|
||||
|
||||
} // namespace rocr
|
||||
|
||||
template <uint32_t lowBit, uint32_t highBit, typename T>
|
||||
static __forceinline uint32_t BitSelect(T p) {
|
||||
static_assert(sizeof(T) <= sizeof(uintptr_t), "Type out of range.");
|
||||
static_assert(highBit < sizeof(uintptr_t) * 8, "Bit index out of range.");
|
||||
|
||||
uintptr_t ptr = p;
|
||||
if (highBit != (sizeof(uintptr_t) * 8 - 1))
|
||||
return (uint32_t)((ptr & ((1ull << (highBit + 1)) - 1)) >> lowBit);
|
||||
else
|
||||
return (uint32_t)(ptr >> lowBit);
|
||||
}
|
||||
|
||||
inline uint32_t PtrLow16Shift8(const void* p) {
|
||||
uintptr_t ptr = reinterpret_cast<uintptr_t>(p);
|
||||
return (uint32_t)((ptr & 0xFFFFULL) >> 8);
|
||||
}
|
||||
|
||||
inline uint32_t PtrHigh64Shift16(const void* p) {
|
||||
uintptr_t ptr = reinterpret_cast<uintptr_t>(p);
|
||||
return (uint32_t)((ptr & 0xFFFFFFFFFFFF0000ULL) >> 16);
|
||||
}
|
||||
|
||||
inline uint32_t PtrLow40Shift8(const void* p) {
|
||||
uintptr_t ptr = reinterpret_cast<uintptr_t>(p);
|
||||
return (uint32_t)((ptr & 0xFFFFFFFFFFULL) >> 8);
|
||||
}
|
||||
|
||||
inline uint32_t PtrHigh64Shift40(const void* p) {
|
||||
uintptr_t ptr = reinterpret_cast<uintptr_t>(p);
|
||||
return (uint32_t)((ptr & 0xFFFFFF0000000000ULL) >> 40);
|
||||
}
|
||||
|
||||
static inline uint8_t Ptr48High8(const void* p) {
|
||||
uintptr_t ptr = reinterpret_cast<uintptr_t>(p);
|
||||
return (uint8_t)((ptr & 0xFF0000000000ULL) >> 40);
|
||||
}
|
||||
|
||||
static inline uint32_t Ptr48Low32(const void* p) {
|
||||
uintptr_t ptr = reinterpret_cast<uintptr_t>(p);
|
||||
assert((ptr & 0xFFFFFFFFFF00ULL) == ptr);
|
||||
return (uint32_t)((ptr & 0xFFFFFFFFFFULL) >> 8);
|
||||
}
|
||||
|
||||
inline uint32_t PtrLow32(const void* p) {
|
||||
return static_cast<uint32_t>(reinterpret_cast<uintptr_t>(p));
|
||||
}
|
||||
|
||||
inline uint32_t PtrHigh32(const void* p) {
|
||||
uint32_t ptr = 0;
|
||||
#ifdef HSA_LARGE_MODEL
|
||||
ptr = static_cast<uint32_t>(reinterpret_cast<uintptr_t>(p) >> 32);
|
||||
#endif
|
||||
return ptr;
|
||||
}
|
||||
|
||||
inline uint32_t HighPart(uint64_t value) {
|
||||
return (value & 0xFFFFFFFF00000000) >> 32;
|
||||
}
|
||||
|
||||
inline uint32_t LowPart(uint64_t value) {
|
||||
return (value & 0x00000000FFFFFFFF);
|
||||
}
|
||||
|
||||
#include "atomic_helpers.h"
|
||||
|
||||
#endif // HSA_RUNTIME_CORE_UTIL_UTILS_H_
|
||||
@@ -0,0 +1,327 @@
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
//
|
||||
// The University of Illinois/NCSA
|
||||
// Open Source License (NCSA)
|
||||
//
|
||||
// Copyright (c) 2014-2020, Advanced Micro Devices, Inc. All rights reserved.
|
||||
//
|
||||
// Developed by:
|
||||
//
|
||||
// AMD Research and AMD HSA Software Development
|
||||
//
|
||||
// Advanced Micro Devices, Inc.
|
||||
//
|
||||
// www.amd.com
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
// of this software and associated documentation files (the "Software"), to
|
||||
// deal with the Software without restriction, including without limitation
|
||||
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
// and/or sell copies of the Software, and to permit persons to whom the
|
||||
// Software is furnished to do so, subject to the following conditions:
|
||||
//
|
||||
// - Redistributions of source code must retain the above copyright notice,
|
||||
// this list of conditions and the following disclaimers.
|
||||
// - Redistributions in binary form must reproduce the above copyright
|
||||
// notice, this list of conditions and the following disclaimers in
|
||||
// the documentation and/or other materials provided with the distribution.
|
||||
// - Neither the names of Advanced Micro Devices, Inc,
|
||||
// nor the names of its contributors may be used to endorse or promote
|
||||
// products derived from this Software without specific prior written
|
||||
// permission.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
|
||||
// THE CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
|
||||
// OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
|
||||
// ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
||||
// DEALINGS WITH THE SOFTWARE.
|
||||
//
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
#ifdef _WIN32 // Are we compiling for windows?
|
||||
#define NOMINMAX
|
||||
|
||||
#include "core/util/os.h"
|
||||
|
||||
#include <algorithm>
|
||||
#include <process.h>
|
||||
#include <string>
|
||||
#include <windows.h>
|
||||
|
||||
#include <emmintrin.h>
|
||||
#include <pmmintrin.h>
|
||||
#include <xmmintrin.h>
|
||||
|
||||
#undef Yield
|
||||
#undef CreateMutex
|
||||
|
||||
namespace rocr {
|
||||
namespace os {
|
||||
|
||||
static_assert(sizeof(LibHandle) == sizeof(HMODULE),
|
||||
"OS abstraction size mismatch");
|
||||
static_assert(sizeof(LibHandle) == sizeof(::HANDLE),
|
||||
"OS abstraction size mismatch");
|
||||
static_assert(sizeof(Semaphore) == sizeof(::HANDLE),
|
||||
"OS abstraction size mismatch");
|
||||
static_assert(sizeof(Mutex) == sizeof(::HANDLE),
|
||||
"OS abstraction size mismatch");
|
||||
static_assert(sizeof(Thread) == sizeof(::HANDLE),
|
||||
"OS abstraction size mismatch");
|
||||
static_assert(sizeof(EventHandle) == sizeof(::HANDLE),
|
||||
"OS abstraction size mismatch");
|
||||
|
||||
LibHandle LoadLib(std::string filename) {
|
||||
HMODULE ret = LoadLibrary(filename.c_str());
|
||||
return *(LibHandle*)&ret;
|
||||
}
|
||||
|
||||
void* GetExportAddress(LibHandle lib, std::string export_name) {
|
||||
return GetProcAddress(*(HMODULE*)&lib, export_name.c_str());
|
||||
}
|
||||
|
||||
void CloseLib(LibHandle lib) { FreeLibrary(*(::HMODULE*)&lib); }
|
||||
|
||||
std::vector<LibHandle> GetLoadedLibs() {
|
||||
// Use EnumProcessModulesEx
|
||||
static_assert(false, "Not implemented.");
|
||||
}
|
||||
|
||||
std::string GetLibraryName(LibHandle lib) {
|
||||
static_assert(false, "Not implemented.");
|
||||
}
|
||||
|
||||
Semaphore CreateSemaphore() {
|
||||
sem = static_cast<void*>(CreateSemaphore(NULL, 0, LONG_MAX, NULL));
|
||||
assert(sem != NULL && "CreateSemaphore failed");
|
||||
|
||||
return *(Semaphore*)&sem;
|
||||
}
|
||||
|
||||
bool WaitSemaphore(Semaphore sem) {
|
||||
return WaitForSingleObject(*(::HANDLE*)&lock, INFINITE) == WAIT_OBJECT_0;
|
||||
}
|
||||
|
||||
void PostSemaphore(Semaphore sem) {
|
||||
ReleaseSemaphore(static_cast<HANDLE>(*sem), 1, NULL);
|
||||
}
|
||||
|
||||
void DestroySemaphore(Semaphore sem) {
|
||||
if (!CloseHandle(static_cast<HANDLE>(*sem))) {
|
||||
assert("CloseHandle() failed");
|
||||
}
|
||||
*sem = NULL;
|
||||
}
|
||||
|
||||
Mutex CreateMutex() { return CreateEvent(NULL, false, true, NULL); }
|
||||
|
||||
bool TryAcquireMutex(Mutex lock) {
|
||||
return WaitForSingleObject(*(::HANDLE*)&lock, 0) == WAIT_OBJECT_0;
|
||||
}
|
||||
|
||||
bool AcquireMutex(Mutex lock) {
|
||||
return WaitForSingleObject(*(::HANDLE*)&lock, INFINITE) == WAIT_OBJECT_0;
|
||||
}
|
||||
|
||||
void ReleaseMutex(Mutex lock) { SetEvent(*(::HANDLE*)&lock); }
|
||||
|
||||
void DestroyMutex(Mutex lock) { CloseHandle(*(::HANDLE*)&lock); }
|
||||
|
||||
void Sleep(int delay_in_millisecond) { ::Sleep(delay_in_millisecond); }
|
||||
|
||||
void uSleep(int delayInUs) { ::Sleep(delayInUs / 1000); }
|
||||
|
||||
void YieldThread() { ::Sleep(0); }
|
||||
|
||||
struct ThreadArgs {
|
||||
void* entry_args;
|
||||
ThreadEntry entry_function;
|
||||
};
|
||||
|
||||
unsigned __stdcall ThreadTrampoline(void* arg) {
|
||||
ThreadArgs* thread_args = (ThreadArgs*)arg;
|
||||
ThreadEntry entry = thread_args->entry_function;
|
||||
void* data = thread_args->entry_args;
|
||||
delete thread_args;
|
||||
entry(data);
|
||||
_endthreadex(0);
|
||||
return 0;
|
||||
}
|
||||
|
||||
Thread CreateThread(ThreadEntry entry_function, void* entry_argument,
|
||||
uint stack_size) {
|
||||
ThreadArgs* thread_args = new ThreadArgs();
|
||||
thread_args->entry_args = entry_argument;
|
||||
thread_args->entry_function = entry_function;
|
||||
uintptr_t ret =
|
||||
_beginthreadex(NULL, stack_size, ThreadTrampoline, thread_args, 0, NULL);
|
||||
return *(Thread*)&ret;
|
||||
}
|
||||
|
||||
void CloseThread(Thread thread) { CloseHandle(*(::HANDLE*)&thread); }
|
||||
|
||||
bool WaitForThread(Thread thread) {
|
||||
return WaitForSingleObject(*(::HANDLE*)&thread, INFINITE) == WAIT_OBJECT_0;
|
||||
}
|
||||
|
||||
bool WaitForAllThreads(Thread* threads, uint thread_count) {
|
||||
return WaitForMultipleObjects(thread_count, threads, TRUE, INFINITE) ==
|
||||
WAIT_OBJECT_0;
|
||||
}
|
||||
|
||||
void SetEnvVar(std::string env_var_name, std::string env_var_value) {
|
||||
SetEnvironmentVariable(env_var_name.c_str(), env_var_value.c_str());
|
||||
}
|
||||
|
||||
std::string GetEnvVar(std::string env_var_name) {
|
||||
char* buff;
|
||||
DWORD char_count = GetEnvironmentVariable(env_var_name.c_str(), NULL, 0);
|
||||
if (char_count == 0) return "";
|
||||
buff = (char*)alloca(sizeof(char) * char_count);
|
||||
GetEnvironmentVariable(env_var_name.c_str(), buff, char_count);
|
||||
buff[char_count - 1] = '\0';
|
||||
std::string ret = buff;
|
||||
return ret;
|
||||
}
|
||||
|
||||
size_t GetUserModeVirtualMemorySize() {
|
||||
SYSTEM_INFO system_info = {0};
|
||||
GetSystemInfo(&system_info);
|
||||
return ((size_t)system_info.lpMaximumApplicationAddress + 1);
|
||||
}
|
||||
|
||||
size_t GetUsablePhysicalHostMemorySize() {
|
||||
MEMORYSTATUSEX memory_status = {0};
|
||||
memory_status.dwLength = sizeof(memory_status);
|
||||
if (GlobalMemoryStatusEx(&memory_status) == 0) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
const size_t physical_size = static_cast<size_t>(memory_status.ullTotalPhys);
|
||||
return std::min(GetUserModeVirtualMemorySize(), physical_size);
|
||||
}
|
||||
|
||||
uintptr_t GetUserModeVirtualMemoryBase() { return (uintptr_t)0; }
|
||||
|
||||
// Os event wrappers
|
||||
EventHandle CreateOsEvent(bool auto_reset, bool init_state) {
|
||||
EventHandle evt = reinterpret_cast<EventHandle>(
|
||||
CreateEvent(NULL, (BOOL)(!auto_reset), (BOOL)init_state, NULL));
|
||||
return evt;
|
||||
}
|
||||
|
||||
int DestroyOsEvent(EventHandle event) {
|
||||
if (event == NULL) {
|
||||
return -1;
|
||||
}
|
||||
return CloseHandle(reinterpret_cast<::HANDLE>(event));
|
||||
}
|
||||
|
||||
int WaitForOsEvent(EventHandle event, unsigned int milli_seconds) {
|
||||
if (event == NULL) {
|
||||
return -1;
|
||||
}
|
||||
|
||||
int ret_code =
|
||||
WaitForSingleObject(reinterpret_cast<::HANDLE>(event), milli_seconds);
|
||||
if (ret_code == WAIT_TIMEOUT) {
|
||||
ret_code = 0x14003; // 0x14003 indicates timeout
|
||||
}
|
||||
return ret_code;
|
||||
}
|
||||
|
||||
int SetOsEvent(EventHandle event) {
|
||||
if (event == NULL) {
|
||||
return -1;
|
||||
}
|
||||
return SetEvent(reinterpret_cast<::HANDLE>(event));
|
||||
}
|
||||
|
||||
int ResetOsEvent(EventHandle event) {
|
||||
if (event == NULL) {
|
||||
return -1;
|
||||
}
|
||||
return ResetEvent(reinterpret_cast<::HANDLE>(event));
|
||||
}
|
||||
|
||||
uint64_t ReadAccurateClock() {
|
||||
uint64_t ret;
|
||||
QueryPerformanceCounter((LARGE_INTEGER*)&ret);
|
||||
return ret;
|
||||
}
|
||||
|
||||
uint64_t AccurateClockFrequency() {
|
||||
uint64_t ret;
|
||||
QueryPerformanceFrequency((LARGE_INTEGER*)&ret);
|
||||
return ret;
|
||||
}
|
||||
|
||||
SharedMutex CreateSharedMutex() {
|
||||
assert(false && "Not implemented.");
|
||||
abort();
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
bool TryAcquireSharedMutex(SharedMutex lock) {
|
||||
assert(false && "Not implemented.");
|
||||
abort();
|
||||
return false;
|
||||
}
|
||||
|
||||
bool AcquireSharedMutex(SharedMutex lock) {
|
||||
assert(false && "Not implemented.");
|
||||
abort();
|
||||
return false;
|
||||
}
|
||||
|
||||
void ReleaseSharedMutex(SharedMutex lock) {
|
||||
assert(false && "Not implemented.");
|
||||
abort();
|
||||
}
|
||||
|
||||
bool TrySharedAcquireSharedMutex(SharedMutex lock) {
|
||||
assert(false && "Not implemented.");
|
||||
abort();
|
||||
return false;
|
||||
}
|
||||
|
||||
bool SharedAcquireSharedMutex(SharedMutex lock) {
|
||||
assert(false && "Not implemented.");
|
||||
abort();
|
||||
return false;
|
||||
}
|
||||
|
||||
void SharedReleaseSharedMutex(SharedMutex lock) {
|
||||
assert(false && "Not implemented.");
|
||||
abort();
|
||||
}
|
||||
|
||||
void DestroySharedMutex(SharedMutex lock) {
|
||||
assert(false && "Not implemented.");
|
||||
abort();
|
||||
}
|
||||
|
||||
uint64_t ReadSystemClock() {
|
||||
assert(false && "Not implemented.");
|
||||
abort();
|
||||
return 0;
|
||||
}
|
||||
|
||||
uint64_t SystemClockFrequency() {
|
||||
assert(false && "Not implemented.");
|
||||
abort();
|
||||
return 0;
|
||||
}
|
||||
|
||||
bool ParseCpuID(cpuid_t* cpuinfo) {
|
||||
assert(false && "Not implemented.");
|
||||
abort();
|
||||
return false;
|
||||
}
|
||||
|
||||
} // namespace os
|
||||
} // namespace rocr
|
||||
|
||||
#endif
|
||||
@@ -0,0 +1,52 @@
|
||||
/*
|
||||
* Copyright © 2014 Advanced Micro Devices, Inc.
|
||||
*
|
||||
* Permission is hereby granted, free of charge, to any person
|
||||
* obtaining a copy of this software and associated documentation
|
||||
* files (the "Software"), to deal in the Software without
|
||||
* restriction, including without limitation the rights to use, copy,
|
||||
* modify, merge, publish, distribute, sublicense, and/or sell copies
|
||||
* of the Software, and to permit persons to whom the Software is
|
||||
* furnished to do so, subject to the following conditions:
|
||||
*
|
||||
* The above copyright notice and this permission notice (including
|
||||
* the next paragraph) shall be included in all copies or substantial
|
||||
* portions of the Software.
|
||||
*
|
||||
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
||||
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
||||
* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
||||
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
|
||||
* HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
|
||||
* WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
||||
* DEALINGS IN THE SOFTWARE.
|
||||
*/
|
||||
|
||||
#include "libhsakmt.h"
|
||||
|
||||
HSAKMT_STATUS HSAKMTAPI hsaKmtGetVersion(HsaVersionInfo *VersionInfo) {
|
||||
CHECK_DXG_OPEN();
|
||||
|
||||
VersionInfo->KernelInterfaceMajorVersion = 1;
|
||||
VersionInfo->KernelInterfaceMinorVersion = 16;
|
||||
|
||||
return HSAKMT_STATUS_SUCCESS;
|
||||
}
|
||||
HSAKMT_STATUS HSAKMTAPI
|
||||
hsaKmtGetVersionCapInfo(HsaVersionCapability *VersionCapInfo) {
|
||||
CHECK_DXG_OPEN();
|
||||
|
||||
VersionCapInfo->Value = 0;
|
||||
/*VersionCapInfo->ui64.InterruptSignal = 0;
|
||||
VersionCapInfo->ui64.Sdma = 0;
|
||||
VersionCapInfo->ui64.SdmaXgmi = 0;
|
||||
VersionCapInfo->ui64.Image = 0;
|
||||
VersionCapInfo->ui64.EventAge = 0;
|
||||
VersionCapInfo->ui64.Scratch = 0;
|
||||
VersionCapInfo->ui64.Vmem = 0;
|
||||
VersionCapInfo->ui64.dmabuf = 0;
|
||||
VersionCapInfo->ui64.XNack = 0;*/
|
||||
|
||||
return HSAKMT_STATUS_SUCCESS;
|
||||
}
|
||||
@@ -0,0 +1,281 @@
|
||||
/* Copyright (c) 2023 Advanced Micro Devices, Inc. All rights reserved. */
|
||||
|
||||
#include "inc/wddm/cmd_util.h"
|
||||
|
||||
namespace rocr {
|
||||
namespace core {
|
||||
|
||||
/*
|
||||
* Builds a COPY_DATA packet that copies data.
|
||||
*/
|
||||
size_t CmdUtil::BuildCopyData(
|
||||
uint64_t *pDstAddr,
|
||||
void *pBuffer,
|
||||
uint32_t dstSel,
|
||||
uint32_t dstCachePolicy,
|
||||
uint32_t srcSel,
|
||||
uint32_t srcCachePolicy,
|
||||
uint32_t countSel,
|
||||
uint32_t wrConfirm) {
|
||||
PM4MEC_COPY_DATA copy_data = {0};
|
||||
|
||||
GenerateCmdHeader(©_data, IT_COPY_DATA);
|
||||
copy_data.bitfields2.dst_sel = dstSel;
|
||||
copy_data.bitfields2.src_sel = srcSel;
|
||||
copy_data.bitfields2.dst_cache_policy = dstCachePolicy;
|
||||
copy_data.bitfields2.src_cache_policy = srcCachePolicy;
|
||||
copy_data.bitfields2.count_sel = countSel;
|
||||
copy_data.bitfields2.wr_confirm = wrConfirm;
|
||||
copy_data.bitfields5c.dst_64b_addr_lo = (PtrLow32(pDstAddr) >> 3);
|
||||
copy_data.dst_addr_hi = PtrHigh32(pDstAddr);
|
||||
memcpy(pBuffer, ©_data, sizeof(copy_data));
|
||||
|
||||
return sizeof(copy_data);
|
||||
}
|
||||
|
||||
/*
|
||||
* Builds a EVENT_WRITE packet.
|
||||
* Applications can use Barrier command to ensure their
|
||||
* command is executed only after all other commands have
|
||||
* completed their execution.
|
||||
*/
|
||||
size_t CmdUtil::BuildBarrier(
|
||||
void *pBuffer,
|
||||
uint32_t eventIndex,
|
||||
uint32_t eventType) {
|
||||
BarrierTemplate barrier = {0};
|
||||
|
||||
GenerateCmdHeader(&barrier.event_write, IT_EVENT_WRITE);
|
||||
barrier.event_write.bitfields2.event_index = eventIndex;
|
||||
barrier.event_write.bitfields2.event_type = eventType;
|
||||
memcpy(pBuffer, &barrier, sizeof(barrier));
|
||||
|
||||
return sizeof(barrier);
|
||||
}
|
||||
|
||||
/*
|
||||
* Builds a ACQUIRE_MEM packet.
|
||||
* Users can submit this command to
|
||||
* invalidate Gpu caches - L1 and or L2.
|
||||
*/
|
||||
size_t CmdUtil::BuildAcquireMem(
|
||||
uint8_t major,
|
||||
void *pBuffer) {
|
||||
size_t ret;
|
||||
if (major == 9) {
|
||||
gfx9::AcquireMemTemplate acq = {0};
|
||||
GenerateCmdHeader(&acq.acquire_mem, IT_ACQUIRE_MEM);
|
||||
// Specify the size of memory to invalidate. Size is
|
||||
// specified in terms of 256 byte chunks. A coher_size
|
||||
// of 0xFFFFFFFF actually specified 0xFFFFFFFF00 (40 bits)
|
||||
// of memory. The field coher_size_hi specifies memory from
|
||||
// bits 40-64 for a total of 256 TB.
|
||||
acq.acquire_mem.coher_size = 0xFFFFFFFF;
|
||||
acq.acquire_mem.bitfields4.coher_size_hi = 0xFF;
|
||||
// Specify the address of memory to invalidate. The
|
||||
// address must be 256 byte aligned.
|
||||
acq.acquire_mem.coher_base_lo = 0;
|
||||
acq.acquire_mem.bitfields6.coher_base_hi = 0;
|
||||
// Specify the poll interval for determing if operation is complete
|
||||
acq.acquire_mem.bitfields7.poll_interval = 4;
|
||||
acq.acquire_mem.bitfields2.coher_cntl =
|
||||
(1 << 29) | // CP_COHER_CNTL__SH_ICACHE_ACTION_ENA_MASK
|
||||
(1 << 27) | // CP_COHER_CNTL__SH_KCACHE_ACTION_ENA_MASK
|
||||
(1 << 28); // CP_COHER_CNTL__SH_KCACHE_VOL_ACTION_ENA_MASK
|
||||
memcpy(pBuffer, &acq, sizeof(acq));
|
||||
ret = sizeof(acq);
|
||||
} else if (major >= 10) {
|
||||
gfx10::AcquireMemTemplate acq = {0};
|
||||
GenerateCmdHeader(&acq.acquire_mem, IT_ACQUIRE_MEM);
|
||||
acq.acquire_mem.coher_size = 0xFFFFFFFF;
|
||||
acq.acquire_mem.bitfields4.coher_size_hi = 0xFF;
|
||||
acq.acquire_mem.coher_base_lo = 0;
|
||||
acq.acquire_mem.bitfields6.coher_base_hi = 0;
|
||||
acq.acquire_mem.bitfields7.poll_interval = 4;
|
||||
acq.acquire_mem.bitfields8.gcr_cntl =
|
||||
(1 << 16) | // SEQ = FORWARD
|
||||
(1 << 15) | // GL2_WB
|
||||
(1 << 14) | // GL2_INV
|
||||
(1 << 9) | // GL1_INV
|
||||
(1 << 8) | // GLV_INV
|
||||
(1 << 7) | // GLK_INV
|
||||
(1 << 6) | // GLK_WB
|
||||
(1 << 5) | // GLM_INV
|
||||
(1 << 4) | // GLM_WB
|
||||
(1 << 0); // GLI_INV = ALL
|
||||
memcpy(pBuffer, &acq, sizeof(acq));
|
||||
ret = sizeof(acq);
|
||||
}
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
/*
|
||||
* Builds a scratch packet.
|
||||
*/
|
||||
size_t CmdUtil::BuildScratch(
|
||||
void *pScratchBase,
|
||||
void *pBuffer) {
|
||||
struct SetScratchTemplate scratch = {0};
|
||||
|
||||
GenerateSetShRegHeader(&scratch, mmCOMPUTE_DISPATCH_SCRATCH_BASE_LO);
|
||||
scratch.scratch_lo = Ptr48Low32(pScratchBase);
|
||||
scratch.scratch_hi = Ptr48High8(pScratchBase);
|
||||
memcpy(pBuffer, &scratch, sizeof(scratch));
|
||||
|
||||
return sizeof(scratch);
|
||||
}
|
||||
|
||||
/**
|
||||
* @ Set Compute Shader parameter for gfx11 and above
|
||||
*/
|
||||
size_t CmdUtil::BuildComputeShaderParams(void *pBuffer) {
|
||||
struct DispatchProgramResourceRegs compute_shader_params = {0};
|
||||
|
||||
GenerateSetShRegHeader(&compute_shader_params, mmCOMPUTE_PGM_RSRC3);
|
||||
// IMAGE_OP: Indicates the compute program contains an image op
|
||||
// instruction and should be stalled by its WAIT_SYNC fence.
|
||||
compute_shader_params.compute_pgm_rsrc3 = (1 << 31);
|
||||
|
||||
memcpy(pBuffer, &compute_shader_params, sizeof(compute_shader_params));
|
||||
|
||||
return sizeof(compute_shader_params);
|
||||
}
|
||||
|
||||
|
||||
/*
|
||||
* Builds a dispatch packet.
|
||||
*/
|
||||
size_t CmdUtil::BuildDispatch(
|
||||
struct DispatchInfo *pInfo,
|
||||
void *pBuffer) {
|
||||
DispatchTemplate dispatch = {0};
|
||||
|
||||
GenerateSetShRegHeader(&dispatch.dimension_regs, mmCOMPUTE_NUM_THREAD_X);
|
||||
dispatch.dimension_regs.compute_num_thread_x = pInfo->pPacket->workgroup_size_x;
|
||||
dispatch.dimension_regs.compute_num_thread_y = pInfo->pPacket->workgroup_size_y;
|
||||
dispatch.dimension_regs.compute_num_thread_z = pInfo->pPacket->workgroup_size_z;
|
||||
|
||||
// TODO: Add AQL packet index for debugger
|
||||
// Debugger requires AQL packet index in COMPUTE_DISPATCH_PKT_ADDR_LO
|
||||
GenerateSetShRegHeader(&dispatch.program_regs, mmCOMPUTE_PGM_LO);
|
||||
dispatch.program_regs.compute_pgm_lo = Ptr48Low32(pInfo->pEntry);
|
||||
dispatch.program_regs.compute_pgm_hi = Ptr48High8(pInfo->pEntry);
|
||||
|
||||
GenerateSetShRegHeader(&dispatch.program_resource_regs, mmCOMPUTE_PGM_RSRC1);
|
||||
dispatch.program_resource_regs.compute_pgm_rsrc1 = pInfo->pKernelObject->compute_pgm_rsrc1;
|
||||
dispatch.program_resource_regs.compute_pgm_rsrc2 =
|
||||
(pInfo->ldsBlks << 15) | pInfo->pKernelObject->compute_pgm_rsrc2;
|
||||
|
||||
GenerateSetShRegHeader(&dispatch.resource_regs, mmCOMPUTE_RESOURCE_LIMITS);
|
||||
dispatch.resource_regs.compute_resource_limits = 0x3ff;
|
||||
dispatch.resource_regs.compute_static_thread_mgmt_se0 = 0xFFFFFFFF;
|
||||
dispatch.resource_regs.compute_static_thread_mgmt_se1 = 0xFFFFFFFF;
|
||||
dispatch.resource_regs.compute_static_thread_mgmt_se2 = 0xFFFFFFFF;
|
||||
dispatch.resource_regs.compute_static_thread_mgmt_se3 = 0xFFFFFFFF;
|
||||
|
||||
dispatch.resource_regs.compute_tmpring_size = pInfo->pAmdQueue->compute_tmpring_size;
|
||||
|
||||
GenerateSetShRegHeader(&dispatch.compute_user_data_regs, mmCOMPUTE_USER_DATA_0);
|
||||
|
||||
uint32_t sgpr_no = 0;
|
||||
if (AMD_HSA_BITS_GET(pInfo->pKernelObject->kernel_code_properties,
|
||||
AMD_KERNEL_CODE_PROPERTIES_ENABLE_SGPR_PRIVATE_SEGMENT_BUFFER)) {
|
||||
assert(pInfo->major < 11);
|
||||
pInfo->scratchBaseOffset[pInfo->offsetCnt++] =
|
||||
offsetof(struct DispatchTemplate, compute_user_data_regs.compute_user_data[0]) +
|
||||
sgpr_no * sizeof(uint32_t);
|
||||
|
||||
dispatch.compute_user_data_regs.compute_user_data[sgpr_no++] =
|
||||
pInfo->pAmdQueue->scratch_resource_descriptor[0];
|
||||
dispatch.compute_user_data_regs.compute_user_data[sgpr_no++] =
|
||||
pInfo->pAmdQueue->scratch_resource_descriptor[1];
|
||||
dispatch.compute_user_data_regs.compute_user_data[sgpr_no++] =
|
||||
pInfo->pAmdQueue->scratch_resource_descriptor[2];
|
||||
dispatch.compute_user_data_regs.compute_user_data[sgpr_no++] =
|
||||
pInfo->srd;
|
||||
}
|
||||
if (AMD_HSA_BITS_GET(pInfo->pKernelObject->kernel_code_properties,
|
||||
AMD_KERNEL_CODE_PROPERTIES_ENABLE_SGPR_DISPATCH_PTR)) {
|
||||
dispatch.compute_user_data_regs.compute_user_data[sgpr_no++] = PtrLow32(pInfo->pPacket);
|
||||
dispatch.compute_user_data_regs.compute_user_data[sgpr_no++] = PtrHigh32(pInfo->pPacket);
|
||||
}
|
||||
if (AMD_HSA_BITS_GET(pInfo->pKernelObject->kernel_code_properties,
|
||||
AMD_KERNEL_CODE_PROPERTIES_ENABLE_SGPR_QUEUE_PTR)) {
|
||||
dispatch.compute_user_data_regs.compute_user_data[sgpr_no++] = PtrLow32(pInfo->pAmdQueue);
|
||||
dispatch.compute_user_data_regs.compute_user_data[sgpr_no++] = PtrHigh32(pInfo->pAmdQueue);
|
||||
}
|
||||
if (AMD_HSA_BITS_GET(pInfo->pKernelObject->kernel_code_properties,
|
||||
AMD_KERNEL_CODE_PROPERTIES_ENABLE_SGPR_KERNARG_SEGMENT_PTR)) {
|
||||
dispatch.compute_user_data_regs.compute_user_data[sgpr_no++] =
|
||||
PtrLow32(pInfo->pPacket->kernarg_address);
|
||||
dispatch.compute_user_data_regs.compute_user_data[sgpr_no++] =
|
||||
PtrHigh32(pInfo->pPacket->kernarg_address);
|
||||
}
|
||||
if (AMD_HSA_BITS_GET(pInfo->pKernelObject->kernel_code_properties,
|
||||
AMD_KERNEL_CODE_PROPERTIES_ENABLE_SGPR_DISPATCH_ID)) {
|
||||
// This feature may be enabled as a side effect of indirect calls.
|
||||
// However, the compiler team confirmed that the dispatch id itself is not used,
|
||||
// so safe to send 0 for each dispatch.
|
||||
dispatch.compute_user_data_regs.compute_user_data[sgpr_no++] = NULL;
|
||||
dispatch.compute_user_data_regs.compute_user_data[sgpr_no++] = NULL;
|
||||
}
|
||||
if (AMD_HSA_BITS_GET(pInfo->pKernelObject->kernel_code_properties,
|
||||
AMD_KERNEL_CODE_PROPERTIES_ENABLE_SGPR_FLAT_SCRATCH_INIT)) {
|
||||
assert(pInfo->major < 11);
|
||||
pInfo->scratchBaseOffset[pInfo->offsetCnt++] =
|
||||
offsetof(struct DispatchTemplate, compute_user_data_regs.compute_user_data[0]) +
|
||||
sgpr_no * sizeof(uint32_t);
|
||||
|
||||
dispatch.compute_user_data_regs.compute_user_data[sgpr_no++] =
|
||||
PtrLow32(pInfo->pScratchBase);
|
||||
dispatch.compute_user_data_regs.compute_user_data[sgpr_no++] =
|
||||
PtrHigh32(pInfo->pScratchBase);
|
||||
}
|
||||
if (AMD_HSA_BITS_GET(pInfo->pKernelObject->kernel_code_properties,
|
||||
AMD_KERNEL_CODE_PROPERTIES_ENABLE_SGPR_PRIVATE_SEGMENT_SIZE)) {
|
||||
dispatch.compute_user_data_regs.compute_user_data[sgpr_no++] =
|
||||
pInfo->scratchSizePerWave / (pInfo->wave32 ? 32 : 64);
|
||||
}
|
||||
|
||||
GenerateCmdHeader(&dispatch.dispatch_direct, IT_DISPATCH_DIRECT);
|
||||
dispatch.dispatch_direct.dispatch_initiator =
|
||||
(1 << 0) | // COMPUTE_SHADER_EN
|
||||
(1 << 2) | // FORCE_START_AT_000
|
||||
(1 << 5); // USE_THREAD_DIMENSIONS
|
||||
if (pInfo->wave32) dispatch.dispatch_direct.dispatch_initiator |= (1 << 15); // CS_W32_EN
|
||||
dispatch.dispatch_direct.dim_x = pInfo->pPacket->grid_size_x;
|
||||
dispatch.dispatch_direct.dim_y = pInfo->pPacket->grid_size_y;
|
||||
dispatch.dispatch_direct.dim_z = pInfo->pPacket->grid_size_z;
|
||||
memcpy(pBuffer, &dispatch, sizeof(dispatch));
|
||||
|
||||
return sizeof(dispatch);
|
||||
}
|
||||
|
||||
/*
|
||||
* Builds a ATOMIC_MEM packet.
|
||||
* Users can submit this command
|
||||
* to perform atomic operations.
|
||||
*/
|
||||
size_t CmdUtil::BuildAtomicMem(
|
||||
uint64_t *pAddr,
|
||||
uint32_t atomic,
|
||||
void *pBuffer,
|
||||
uint32_t cachePolicy,
|
||||
uint64_t srcData) {
|
||||
AtomicTemplate atom = {0};
|
||||
|
||||
GenerateCmdHeader(&atom.atomic, IT_ATOMIC_MEM);
|
||||
atom.atomic.addr_lo = PtrLow32(pAddr);
|
||||
atom.atomic.addr_hi = PtrHigh32(pAddr);
|
||||
atom.atomic.bitfields2.atomic = atomic;
|
||||
atom.atomic.bitfields2.cache_policy = cachePolicy;
|
||||
atom.atomic.src_data_lo = LowPart(srcData);
|
||||
atom.atomic.src_data_hi = HighPart(srcData);
|
||||
memcpy(pBuffer, &atom, sizeof(atom));
|
||||
|
||||
return sizeof(atom);
|
||||
}
|
||||
|
||||
} // namespace core
|
||||
} // namespace rocr
|
||||
@@ -0,0 +1,879 @@
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
//
|
||||
// The University of Illinois/NCSA
|
||||
// Open Source License (NCSA)
|
||||
//
|
||||
// Copyright (c) 2020, Advanced Micro Devices, Inc. All rights reserved.
|
||||
//
|
||||
// Developed by:
|
||||
//
|
||||
// AMD Research and AMD HSA Software Development
|
||||
//
|
||||
// Advanced Micro Devices, Inc.
|
||||
//
|
||||
// www.amd.com
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
// of this software and associated documentation files (the "Software"), to
|
||||
// deal with the Software without restriction, including without limitation
|
||||
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
// and/or sell copies of the Software, and to permit persons to whom the
|
||||
// Software is furnished to do so, subject to the following conditions:
|
||||
//
|
||||
// - Redistributions of source code must retain the above copyright notice,
|
||||
// this list of conditions and the following disclaimers.
|
||||
// - Redistributions in binary form must reproduce the above copyright
|
||||
// notice, this list of conditions and the following disclaimers in
|
||||
// the documentation and/or other materials provided with the distribution.
|
||||
// - Neither the names of Advanced Micro Devices, Inc,
|
||||
// nor the names of its contributors may be used to endorse or promote
|
||||
// products derived from this Software without specific prior written
|
||||
// permission.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
|
||||
// THE CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
|
||||
// OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
|
||||
// ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
||||
// DEALINGS WITH THE SOFTWARE.
|
||||
//
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
#include <cinttypes>
|
||||
#include <bitset>
|
||||
|
||||
#include <sys/mman.h>
|
||||
#include <sys/sysinfo.h>
|
||||
#include <linux/mman.h>
|
||||
#include <fcntl.h>
|
||||
#include <unistd.h>
|
||||
#include "inc/wddm/status.h"
|
||||
#include "inc/wddm/types.h"
|
||||
#include "inc/wddm/device.h"
|
||||
#include "inc/wddm/queue.h"
|
||||
|
||||
namespace rocr {
|
||||
namespace core {
|
||||
|
||||
const uint32_t WDDMDevice::cmdbuf_aql_frame_num_ = 0x1000;
|
||||
|
||||
WDDMDevice::WDDMDevice(D3DKMT_HANDLE adapter, LUID adapter_luid)
|
||||
: adapter_(adapter), adapter_luid_(adapter_luid) {
|
||||
memset(&device_info_, 0, sizeof(device_info_));
|
||||
|
||||
ParseDeviceInfo();
|
||||
CreateDevice();
|
||||
SetPowerOptimization(false);
|
||||
CreatePagingQueue();
|
||||
ReserveLocalHeapSpace();
|
||||
ReserveSystemHeapSpace();
|
||||
InitVaMgr();
|
||||
InitCmdbufInfo();
|
||||
}
|
||||
|
||||
WDDMDevice::~WDDMDevice() {
|
||||
FreeLocalHeapSpace();
|
||||
FreeSystemHeapSpace();
|
||||
DestroyPagingQueue();
|
||||
SetPowerOptimization(true);
|
||||
DestroyDevice();
|
||||
|
||||
DestroyDeviceInfo();
|
||||
}
|
||||
|
||||
static NTSTATUS WDDMQueryAdapter(D3DKMT_HANDLE adapter, KMTQUERYADAPTERINFOTYPE type,
|
||||
void *data, int size)
|
||||
{
|
||||
D3DKMT_QUERYADAPTERINFO args = {0};
|
||||
|
||||
args.hAdapter = adapter;
|
||||
args.Type = type;
|
||||
args.pPrivateDriverData = data;
|
||||
args.PrivateDriverDataSize = size;
|
||||
|
||||
return D3DKMTQueryAdapterInfo(&args);
|
||||
}
|
||||
|
||||
uint64_t WDDMDevice::VramAvail(void) {
|
||||
D3DKMT_QUERYSTATISTICS stats;
|
||||
NTSTATUS ret;
|
||||
uint64_t usedVis = 0;
|
||||
uint64_t usedInv = 0;
|
||||
|
||||
// wait fence complete
|
||||
uint64_t value = page_fence_value_.load();
|
||||
if(!CpuWait(&page_syncobj_, &value, 1, false))
|
||||
return HSA_STATUS_ERROR;
|
||||
|
||||
// local cpu-visible memory
|
||||
memset(&stats, 0, sizeof(D3DKMT_QUERYSTATISTICS));
|
||||
stats.Type = D3DKMT_QUERYSTATISTICS_SEGMENT;
|
||||
stats.AdapterLuid = adapter_luid_;
|
||||
stats.QuerySegment.SegmentId = 0;
|
||||
ret = D3DKMTQueryStatistics(&stats);
|
||||
if (ret == 0)
|
||||
usedVis = stats.QueryResult.SegmentInformation.BytesResident;
|
||||
|
||||
// local invisible memory
|
||||
memset(&stats, 0, sizeof(D3DKMT_QUERYSTATISTICS));
|
||||
stats.Type = D3DKMT_QUERYSTATISTICS_SEGMENT;
|
||||
stats.AdapterLuid = adapter_luid_;
|
||||
stats.QuerySegment.SegmentId = 1;
|
||||
|
||||
ret = D3DKMTQueryStatistics(&stats);
|
||||
if (ret == 0)
|
||||
usedInv = stats.QueryResult.SegmentInformation.BytesResident;
|
||||
|
||||
return LocalHeapSize() - usedVis - usedInv;
|
||||
}
|
||||
|
||||
bool WDDMDevice::CreateDevice(void) {
|
||||
D3DKMT_CREATEDEVICE args = {0};
|
||||
args.hAdapter = adapter_;
|
||||
|
||||
NTSTATUS ret = D3DKMTCreateDevice(&args);
|
||||
if (ret == STATUS_SUCCESS) {
|
||||
device_ = args.hDevice;
|
||||
return true;
|
||||
}
|
||||
|
||||
fprintf(stderr, "%s fail %x\n", __FUNCTION__, ret);
|
||||
return false;
|
||||
}
|
||||
|
||||
bool WDDMDevice::DestroyDevice(void) {
|
||||
D3DKMT_DESTROYDEVICE args = {0};
|
||||
args.hDevice = device_;
|
||||
|
||||
NTSTATUS ret = D3DKMTDestroyDevice(&args);
|
||||
if (ret == STATUS_SUCCESS)
|
||||
return true;
|
||||
|
||||
fprintf(stderr, "%s fail %x\n", __FUNCTION__, ret);
|
||||
return false;
|
||||
}
|
||||
|
||||
bool WDDMDevice::CreatePagingQueue(void) {
|
||||
D3DKMT_CREATEPAGINGQUEUE args = {0};
|
||||
args.hDevice = device_;
|
||||
args.Priority = D3DDDI_PAGINGQUEUE_PRIORITY_NORMAL;
|
||||
|
||||
NTSTATUS ret = D3DKMTCreatePagingQueue(&args);
|
||||
if (ret == STATUS_SUCCESS) {
|
||||
page_queue_ = args.hPagingQueue;
|
||||
page_syncobj_ = args.hSyncObject;
|
||||
page_fence_addr_ = (uint64_t *)args.FenceValueCPUVirtualAddress;
|
||||
page_fence_value_ = 0;
|
||||
return true;
|
||||
}
|
||||
|
||||
fprintf(stderr, "%s fail %x\n", __FUNCTION__, ret);
|
||||
return false;
|
||||
}
|
||||
|
||||
bool WDDMDevice::DestroyPagingQueue(void) {
|
||||
D3DDDI_DESTROYPAGINGQUEUE args = {0};
|
||||
args.hPagingQueue = page_queue_;
|
||||
|
||||
NTSTATUS ret = D3DKMTDestroyPagingQueue(&args);
|
||||
if (ret == STATUS_SUCCESS)
|
||||
return true;
|
||||
|
||||
fprintf(stderr, "%s fail %x\n", __FUNCTION__, ret);
|
||||
return false;
|
||||
}
|
||||
|
||||
bool WDDMDevice::CommitSystemHeapSpace(void* addr, int64_t size, bool lock) {
|
||||
int32_t protFlags = PROT_READ | PROT_WRITE | PROT_EXEC;
|
||||
int32_t mapFlags = MAP_PRIVATE|MAP_ANONYMOUS|MAP_FIXED|
|
||||
MAP_NORESERVE|MAP_UNINITIALIZED;
|
||||
if (lock)
|
||||
mapFlags |= MAP_LOCKED;
|
||||
void* paddr = mmap(addr, size, protFlags, mapFlags, -1, 0);
|
||||
if (paddr == MAP_FAILED) {
|
||||
fprintf(stderr, "%s fail to commit %s addr = %p, paddr = %p\n",
|
||||
__FUNCTION__, (lock ? "locked" : ""), addr, paddr);
|
||||
return false;
|
||||
}
|
||||
assert(addr == paddr);
|
||||
|
||||
/*if (!Runtime::runtime_singleton_->PinWARequired())
|
||||
return true;*/
|
||||
|
||||
/*
|
||||
* Do not make the pages in this range available to the child
|
||||
* after a fork(2). This is useful to prevent copy-on-write
|
||||
* semantics from changing the physical location of a page if
|
||||
* the parent writes to it after a fork(2). (Such page
|
||||
* relocations cause problems for hardware that DMAs into the
|
||||
* page.)
|
||||
*
|
||||
* https://man7.org/linux/man-pages/man2/madvise.2.html
|
||||
*/
|
||||
if (madvise(addr, size, MADV_DONTFORK))
|
||||
fprintf(stderr, "%s fail to set MADV_DONTFORK for addr = %p\n",
|
||||
__FUNCTION__, addr);
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
bool WDDMDevice::DecommitSystemHeapSpace(void* addr, int64_t size) {
|
||||
int32_t protFlags = PROT_NONE;
|
||||
int32_t mapFlags = MAP_PRIVATE|MAP_ANONYMOUS|MAP_FIXED|
|
||||
MAP_NORESERVE|MAP_UNINITIALIZED;
|
||||
void* paddr = mmap(addr, size, protFlags, mapFlags, -1, 0);
|
||||
if (paddr == MAP_FAILED) {
|
||||
fprintf(stderr, "%s fail to decommit addr = %p, paddr = %p\n",
|
||||
__FUNCTION__, addr, paddr);
|
||||
return false;
|
||||
}
|
||||
assert(addr == paddr);
|
||||
return true;
|
||||
}
|
||||
|
||||
bool WDDMDevice::ReserveSystemHeapSpace() {
|
||||
struct sysinfo info;
|
||||
int ret = sysinfo(&info);
|
||||
uint64_t max_ram = 0x10000000000;
|
||||
uint64_t alignment = 0x100000000;
|
||||
assert(!ret);
|
||||
|
||||
int32_t protFlags = PROT_NONE;
|
||||
// minimum of reserve size is 8G, maximum of reserve size is 1T.
|
||||
system_heap_space_size_ = std::min(AlignUp(info.totalram, alignment) * 2, max_ram);
|
||||
void* cpu = mmap(NULL, system_heap_space_size_, protFlags,
|
||||
MAP_PRIVATE|MAP_ANONYMOUS, -1, 0);
|
||||
if (cpu == MAP_FAILED) {
|
||||
fprintf(stderr, "%s fail to reserve system_heap_space_size_ = %lx \n",
|
||||
__FUNCTION__, system_heap_space_size_);
|
||||
return false;
|
||||
}
|
||||
|
||||
system_heap_space_start_ = (uint64_t)cpu;
|
||||
return true;
|
||||
}
|
||||
|
||||
bool WDDMDevice::FreeSystemHeapSpace(void) {
|
||||
void *cpu = (void *)system_heap_space_start_;
|
||||
if (munmap(cpu, system_heap_space_size_ != 0)) {
|
||||
fprintf(stderr, "%s fail to unmap = %p \n", __FUNCTION__, cpu);
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
/*
|
||||
* To find the avaliable same range for cpu
|
||||
* virtual space and gpu virtual space.
|
||||
* sys_va_size of cpu va range is larger 1G
|
||||
* than gpu va range, otherwise ReserveGPUVirtualAddress
|
||||
* will return error.
|
||||
*/
|
||||
bool WDDMDevice::ReserveLocalHeapSpace(void) {
|
||||
uint64_t sys_va[16] = {0};
|
||||
uint64_t local_va;
|
||||
uint64_t sys_va_size;
|
||||
int match_index = -1;
|
||||
uint64_t align = 0x40000000; /* 1G */
|
||||
void* ptr = NULL;
|
||||
|
||||
local_heap_space_start_ = 0;
|
||||
local_heap_space_size_ = AlignUp(LocalHeapSize(), align) * 4;
|
||||
sys_va_size = local_heap_space_size_ + align;
|
||||
|
||||
/* it will retry 16 times to find the avaliable range. */
|
||||
for (int i = 0; i < 16; i++) {
|
||||
local_va = 0;
|
||||
ptr = mmap(NULL, sys_va_size , PROT_NONE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0);
|
||||
if (ptr == MAP_FAILED) {
|
||||
fprintf(stderr, "%s fail to reserve cpu va in %d time!\n",
|
||||
__FUNCTION__, i);
|
||||
break;
|
||||
}
|
||||
|
||||
sys_va[i] = (uint64_t)ptr;
|
||||
|
||||
if (thunk::ReserveGpuVirtualAddress(
|
||||
adapter_, local_heap_space_size_,
|
||||
(uint64_t)ptr,
|
||||
(uint64_t)ptr + sys_va_size, &local_va) == ErrorCode::Success) {
|
||||
|
||||
match_index = i;
|
||||
local_heap_space_start_ = local_va;
|
||||
debug_print("%s success to reserve gpu va %lx and va cpu %p in %d time\n",
|
||||
__FUNCTION__, local_va, ptr, i);
|
||||
break;
|
||||
} else {
|
||||
debug_print("%s fail to reserve gpu va for cpu va %p in %d time!\n",
|
||||
__FUNCTION__, ptr, i);
|
||||
}
|
||||
}
|
||||
|
||||
if (match_index >= 0) {
|
||||
/* release cpu unused ranges*/
|
||||
uint64_t left_size = local_va - sys_va[match_index];
|
||||
uint64_t right_size = align - left_size;
|
||||
if ((left_size > 0) && munmap((void*)sys_va[match_index], left_size))
|
||||
fprintf(stderr, "%s fail to unmap left %lx with size %lx\n",
|
||||
__FUNCTION__, sys_va[match_index], left_size);
|
||||
if ((right_size > 0) && munmap((void*)(local_va + local_heap_space_size_), right_size))
|
||||
fprintf(stderr, "%s fail to unmap right %lx with size %lx\n",
|
||||
__FUNCTION__, (local_va + local_heap_space_size_), right_size);
|
||||
} else {
|
||||
fprintf(stderr, "%s fail to reserve Local Heap Space !\n",
|
||||
__FUNCTION__);
|
||||
}
|
||||
|
||||
/* free match fail address for cpu va */
|
||||
int free = match_index >= 0 ? match_index : 16;
|
||||
for (int j = 0; j < free; j++) {
|
||||
if (sys_va[j] != 0 && munmap((void*)sys_va[j], sys_va_size)) {
|
||||
fprintf(stderr, "%s fail to unmap %d %lx\n", __FUNCTION__, j, sys_va[j]);
|
||||
}
|
||||
}
|
||||
|
||||
return match_index >= 0;
|
||||
}
|
||||
|
||||
bool WDDMDevice::FreeLocalHeapSpace(void) {
|
||||
thunk::FreeGpuVirtualAddress(adapter_, local_heap_space_start_, local_heap_space_size_);
|
||||
void *cpu = (void *)local_heap_space_start_;
|
||||
return munmap(cpu, local_heap_space_size_) == 0;
|
||||
}
|
||||
|
||||
void WDDMDevice::InitVaMgr() {
|
||||
uint32_t min_align = 4096;
|
||||
local_va_mgr_ = std::make_unique<VaMgr>(local_heap_space_start_, local_heap_space_size_, min_align);
|
||||
}
|
||||
|
||||
void WDDMDevice::SetPowerOptimization(bool restore) {
|
||||
void *priv_data;
|
||||
int priv_size;
|
||||
|
||||
priv_size = rocr_proxy::CreatePowerOptPrivData(&priv_data, restore);
|
||||
|
||||
D3DKMT_ESCAPE d3dkmt_escape;
|
||||
memset(&d3dkmt_escape, 0, sizeof(d3dkmt_escape));
|
||||
|
||||
d3dkmt_escape.hAdapter = adapter_;
|
||||
d3dkmt_escape.hDevice = device_;
|
||||
d3dkmt_escape.hContext = 0; //KMD only use device to identify the process
|
||||
d3dkmt_escape.Type = D3DKMT_ESCAPE_DRIVERPRIVATE;
|
||||
d3dkmt_escape.pPrivateDriverData = priv_data;
|
||||
d3dkmt_escape.PrivateDriverDataSize = priv_size;
|
||||
d3dkmt_escape.Flags.HardwareAccess = true;
|
||||
|
||||
NTSTATUS status = D3DKMTEscape(&d3dkmt_escape);
|
||||
debug_print("%s status %d restore %d\n", __FUNCTION__, status, restore);
|
||||
rocr_proxy::DestroyPrivData(priv_data);
|
||||
}
|
||||
|
||||
ErrorCode WDDMDevice::ReserveGpuVirtualAddress(const rocr_proxy::AllocDomain domain,
|
||||
gpusize hit_base_addr, gpusize size,
|
||||
gpusize *out_gpu_virt_addr, gpusize alignment, bool lock) {
|
||||
gpusize gpu_addr = 0;
|
||||
ErrorCode code = ErrorCode::Success;
|
||||
|
||||
if (domain == rocr_proxy::kSystem) {
|
||||
|
||||
code = thunk::ReserveGpuVirtualAddress(adapter_, size,
|
||||
system_heap_space_start_,
|
||||
system_heap_space_start_ + system_heap_space_size_,
|
||||
&gpu_addr);
|
||||
if (code != ErrorCode::Success)
|
||||
return code;
|
||||
|
||||
if (!CommitSystemHeapSpace((void*)gpu_addr, size, lock)) {
|
||||
thunk::FreeGpuVirtualAddress(adapter_, gpu_addr, size);
|
||||
code = ErrorCode::SyscallFail;
|
||||
}
|
||||
} else {
|
||||
uint64_t align = alignment == 0 ? (64 * 1024) : alignment; // default 64K alignment
|
||||
if (domain == rocr_proxy::kLocal && size >= GPU_HUGE_PAGE_SIZE)
|
||||
align = GPU_HUGE_PAGE_SIZE;
|
||||
|
||||
gpu_addr = local_va_mgr_->Alloc(size, align, hit_base_addr);
|
||||
if (gpu_addr == 0)
|
||||
code = ErrorCode::OutOfGpuMemory;
|
||||
|
||||
}
|
||||
|
||||
*out_gpu_virt_addr = (code == ErrorCode::Success) ? gpu_addr : 0;
|
||||
return code;
|
||||
}
|
||||
|
||||
ErrorCode WDDMDevice::FreeGpuVirtualAddress(const rocr_proxy::AllocDomain domain,
|
||||
gpusize gpu_addr, gpusize size) {
|
||||
auto code = ErrorCode::Success;
|
||||
|
||||
if (domain == rocr_proxy::kSystem) {
|
||||
|
||||
DecommitSystemHeapSpace((void *)gpu_addr, size);
|
||||
|
||||
thunk::FreeGpuVirtualAddressArgs free_args{};
|
||||
free_args.hAdapter = adapter_;
|
||||
free_args.BaseAddress = gpu_addr;
|
||||
free_args.Size = size;
|
||||
|
||||
code = thunk::FreeGpuVirtualAddress(&free_args);
|
||||
} else {
|
||||
local_va_mgr_->Free(gpu_addr);
|
||||
}
|
||||
|
||||
return code;
|
||||
}
|
||||
|
||||
void WDDMDevice::UpdatePageFence(uint64_t fence_value) {
|
||||
uint64_t current = page_fence_value_.load();
|
||||
|
||||
// atomically set fence value when target is bigger than current one
|
||||
do {
|
||||
if (current >= fence_value)
|
||||
break;
|
||||
} while (!page_fence_value_.compare_exchange_weak(current, fence_value));
|
||||
}
|
||||
|
||||
ErrorCode WDDMDevice::CreateGpuMemory(const GpuMemoryCreateInfo &create_info, GpuMemory **gpu_mem) {
|
||||
ErrorCode ret;
|
||||
|
||||
*gpu_mem = nullptr;
|
||||
auto mem = new GpuMemory(this);
|
||||
if (create_info.dmabuf_fd > 0)
|
||||
ret = mem->ImportPhysicalHandle(create_info.dmabuf_fd);
|
||||
else
|
||||
ret = mem->Init(create_info);
|
||||
if (ret == ErrorCode::Success)
|
||||
*gpu_mem = mem;
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
void *WDDMDevice::Lock(D3DKMT_HANDLE handle) {
|
||||
D3DKMT_LOCK2 args = {0};
|
||||
args.hDevice = device_;
|
||||
args.hAllocation = handle;
|
||||
|
||||
NTSTATUS ret = D3DKMTLock2(&args);
|
||||
if (ret == STATUS_SUCCESS)
|
||||
return args.pData;
|
||||
|
||||
fprintf(stderr, "%s fail %x\n", __FUNCTION__, ret);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
bool WDDMDevice::Unlock(D3DKMT_HANDLE handle) {
|
||||
D3DKMT_UNLOCK2 args = {0};
|
||||
args.hDevice = device_;
|
||||
args.hAllocation = handle;
|
||||
|
||||
NTSTATUS ret = D3DKMTUnlock2(&args);
|
||||
if (ret == STATUS_SUCCESS)
|
||||
return true;
|
||||
|
||||
fprintf(stderr, "%s fail %x\n", __FUNCTION__, ret);
|
||||
return false;
|
||||
}
|
||||
|
||||
bool WDDMDevice::CreateContext(int engine, D3DKMT_HANDLE *handle) {
|
||||
void *priv_data;
|
||||
int priv_size;
|
||||
|
||||
int ordinal = EngineOrdinal(engine, &device_info_);
|
||||
if (ordinal < 0)
|
||||
return false;
|
||||
|
||||
bool FwManagedGfxState = SupportStateShadowingByCpFw();
|
||||
priv_size = rocr_proxy::CreateContextPrivData(&priv_data, FwManagedGfxState);
|
||||
|
||||
D3DKMT_CREATECONTEXTVIRTUAL args = {0};
|
||||
args.hDevice = device_;
|
||||
args.EngineAffinity = 1 << 0;
|
||||
args.NodeOrdinal = ordinal;
|
||||
args.pPrivateDriverData = priv_data;
|
||||
args.PrivateDriverDataSize = priv_size;
|
||||
args.ClientHint = D3DKMT_CLIENTHINT_OPENCL;
|
||||
|
||||
if (IsHwsEnabled(engine))
|
||||
args.Flags.HwQueueSupported = 1;
|
||||
else
|
||||
args.Flags.DisableGpuTimeout = rocr_proxy::ShouldDisableGpuTimeout(engine, &device_info_);
|
||||
|
||||
NTSTATUS ret = D3DKMTCreateContextVirtual(&args);
|
||||
if (ret == STATUS_SUCCESS) {
|
||||
*handle = args.hContext;
|
||||
rocr_proxy::DestroyPrivData(priv_data);
|
||||
return true;
|
||||
}
|
||||
|
||||
rocr_proxy::DestroyPrivData(priv_data);
|
||||
|
||||
fprintf(stderr, "%s fail %x\n", __FUNCTION__, ret);
|
||||
return false;
|
||||
}
|
||||
|
||||
bool WDDMDevice::DestroyContext(D3DKMT_HANDLE handle) {
|
||||
D3DKMT_DESTROYCONTEXT args = {0};
|
||||
args.hContext = handle;
|
||||
|
||||
NTSTATUS ret = D3DKMTDestroyContext(&args);
|
||||
if (ret == STATUS_SUCCESS)
|
||||
return true;
|
||||
|
||||
fprintf(stderr, "%s fail %x\n", __FUNCTION__, ret);
|
||||
return false;
|
||||
}
|
||||
|
||||
bool WDDMDevice::GpuWait(WDDMQueue *queue, const D3DKMT_HANDLE *syncobjs,
|
||||
uint64_t *values, int count) {
|
||||
|
||||
D3DKMT_WAITFORSYNCHRONIZATIONOBJECTFROMGPU args = {0};
|
||||
args.hContext = queue->context;
|
||||
args.ObjectCount = count;
|
||||
args.ObjectHandleArray = syncobjs;
|
||||
args.MonitoredFenceValueArray = values;
|
||||
|
||||
NTSTATUS ret = D3DKMTWaitForSynchronizationObjectFromGpu(&args);
|
||||
if (ret == STATUS_SUCCESS)
|
||||
return true;
|
||||
|
||||
fprintf(stderr, "%s fail %x\n", __FUNCTION__, ret);
|
||||
return false;
|
||||
}
|
||||
|
||||
bool WDDMDevice::GpuSignal(D3DKMT_HANDLE context, const D3DKMT_HANDLE *syncobjs,
|
||||
uint64_t *value, int count) {
|
||||
D3DKMT_SIGNALSYNCHRONIZATIONOBJECTFROMGPU args = {0};
|
||||
args.hContext = context;
|
||||
args.ObjectCount = count;
|
||||
args.ObjectHandleArray = syncobjs;
|
||||
args.MonitoredFenceValueArray = value;
|
||||
|
||||
NTSTATUS ret = D3DKMTSignalSynchronizationObjectFromGpu(&args);
|
||||
if (ret == STATUS_SUCCESS)
|
||||
return true;
|
||||
|
||||
fprintf(stderr, "%s fail %x\n", __FUNCTION__, ret);
|
||||
return false;
|
||||
}
|
||||
|
||||
bool WDDMDevice::CpuWait(const D3DKMT_HANDLE *syncobjs, uint64_t *value,
|
||||
int count, bool wait_any) {
|
||||
D3DKMT_WAITFORSYNCHRONIZATIONOBJECTFROMCPU args = {0};
|
||||
args.hDevice = device_;
|
||||
args.ObjectCount = count;
|
||||
args.ObjectHandleArray = syncobjs;
|
||||
args.FenceValueArray = value;
|
||||
args.Flags.WaitAny = wait_any;
|
||||
|
||||
NTSTATUS ret = D3DKMTWaitForSynchronizationObjectFromCpu(&args);
|
||||
if (ret == STATUS_SUCCESS)
|
||||
return true;
|
||||
|
||||
fprintf(stderr, "%s fail %x\n", __FUNCTION__, ret);
|
||||
return false;
|
||||
}
|
||||
|
||||
bool WDDMDevice::WaitOnPagingFenceFromCpu() {
|
||||
uint64_t page_fence_value = 0;
|
||||
|
||||
page_fence_value = page_fence_value_.load();
|
||||
if (CpuWait(&page_syncobj_, &page_fence_value, 1, false))
|
||||
return true;
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
bool WDDMDevice::CreateSyncobj(D3DKMT_HANDLE *handle, uint64_t **addr) {
|
||||
D3DKMT_CREATESYNCHRONIZATIONOBJECT2 args = {0};
|
||||
args.hDevice = device_;
|
||||
args.Info.Type = D3DDDI_MONITORED_FENCE;
|
||||
args.Info.MonitoredFence.EngineAffinity = 1 << 0;
|
||||
|
||||
NTSTATUS ret = D3DKMTCreateSynchronizationObject2(&args);
|
||||
if (ret == STATUS_SUCCESS) {
|
||||
*handle = args.hSyncObject;
|
||||
*addr = (uint64_t *)args.Info.MonitoredFence.FenceValueCPUVirtualAddress;
|
||||
debug_print("create syncobj cpu addr=%p gpu addr=%" PRIx64 "\n",
|
||||
args.Info.MonitoredFence.FenceValueCPUVirtualAddress,
|
||||
args.Info.MonitoredFence.FenceValueGPUVirtualAddress);
|
||||
return true;
|
||||
}
|
||||
|
||||
fprintf(stderr, "%s fail %x\n", __FUNCTION__, ret);
|
||||
return false;
|
||||
}
|
||||
|
||||
void WDDMDevice::DestroySyncobj(D3DKMT_HANDLE handle) {
|
||||
D3DKMT_DESTROYSYNCHRONIZATIONOBJECT args = {0};
|
||||
args.hSyncObject = handle;
|
||||
|
||||
NTSTATUS ret = D3DKMTDestroySynchronizationObject(&args);
|
||||
if (ret != STATUS_SUCCESS)
|
||||
fprintf(stderr, "%s fail %x\n", __FUNCTION__, ret);
|
||||
}
|
||||
|
||||
void WDDMDevice::InitCmdbufInfo(void) {
|
||||
if (device_info_.major == 9) {
|
||||
cmdbuf_aql_frame_size_ = 2 * sizeof(gfx9::AcquireMemTemplate);
|
||||
} else if (device_info_.major >= 10) {
|
||||
cmdbuf_aql_frame_size_ = 2 * sizeof(gfx10::AcquireMemTemplate);
|
||||
}
|
||||
|
||||
if (device_info_.major >= 11)
|
||||
cmdbuf_aql_frame_size_ += sizeof(SetScratchTemplate);
|
||||
|
||||
cmdbuf_aql_frame_size_ +=
|
||||
sizeof(PM4MEC_COPY_DATA) * 2 +
|
||||
sizeof(BarrierTemplate) * 2 +
|
||||
sizeof(DispatchTemplate) +
|
||||
sizeof(AtomicTemplate) * 2;
|
||||
cmdbuf_aql_frame_size_ = AlignUp(cmdbuf_aql_frame_size_, 0x10);
|
||||
|
||||
cmdbuf_size_ = AlignUp(cmdbuf_aql_frame_num_ * cmdbuf_aql_frame_size_, 0x1000);
|
||||
}
|
||||
|
||||
uint32_t WDDMDevice::LdsBlocks(const hsa_kernel_dispatch_packet_t *pkt) {
|
||||
static const uint32_t blk_sz = 512;
|
||||
uint32_t total_sz = pkt->group_segment_size;
|
||||
uint32_t blk_num = (total_sz + blk_sz - 1) / blk_sz;
|
||||
return blk_num;
|
||||
}
|
||||
|
||||
NTSTATUS WDDMGetAdapters(D3DKMT_ADAPTERINFO *&adapters, int &num_adapters)
|
||||
{
|
||||
bool supported = false;
|
||||
D3DKMT_ENUMADAPTERS2 args = {0};
|
||||
NTSTATUS ret = D3DKMTEnumAdapters2(&args);
|
||||
if (ret != STATUS_SUCCESS)
|
||||
return ret;
|
||||
|
||||
if (!args.NumAdapters) {
|
||||
adapters = NULL;
|
||||
num_adapters = 0;
|
||||
return STATUS_SUCCESS;
|
||||
}
|
||||
|
||||
D3DKMT_ADAPTERINFO *info = new D3DKMT_ADAPTERINFO[args.NumAdapters];
|
||||
if (!info)
|
||||
return STATUS_NO_MEMORY;
|
||||
|
||||
args.pAdapters = info;
|
||||
ret = D3DKMTEnumAdapters2(&args);
|
||||
if (ret != STATUS_SUCCESS)
|
||||
goto err_out0;
|
||||
|
||||
adapters = new D3DKMT_ADAPTERINFO[args.NumAdapters];
|
||||
if (!adapters)
|
||||
goto err_out0;
|
||||
|
||||
num_adapters = 0;
|
||||
for (int i = 0; i < args.NumAdapters; i++) {
|
||||
D3DKMT_ADAPTERREGISTRYINFO query = {0};
|
||||
|
||||
ret = WDDMQueryAdapter(info[i].hAdapter, KMTQAITYPE_ADAPTERREGISTRYINFO,
|
||||
&query, sizeof(query));
|
||||
if (ret != STATUS_SUCCESS)
|
||||
goto err_out1;
|
||||
|
||||
supported = rocr_proxy::QueryAdapterSupported(info[i].hAdapter);
|
||||
|
||||
if (std::wcsstr(query.ChipType, L"AMD") && supported) {
|
||||
adapters[num_adapters++] = info[i];
|
||||
}
|
||||
}
|
||||
|
||||
delete info;
|
||||
return STATUS_SUCCESS;
|
||||
|
||||
err_out1:
|
||||
delete adapters;
|
||||
adapters = NULL;
|
||||
err_out0:
|
||||
delete info;
|
||||
return ret;
|
||||
}
|
||||
|
||||
bool WDDMDevice::ParseDeviceInfo() {
|
||||
bool ret;
|
||||
|
||||
memset(&device_info_, 0, sizeof(device_info_));
|
||||
ret = rocr_proxy::ParseAdapterInfo(adapter_, &device_info_);
|
||||
if (!ret)
|
||||
return false;
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
void WDDMDevice::DestroyDeviceInfo() {
|
||||
free(device_info_.adapter_info);
|
||||
}
|
||||
|
||||
void WDDMDevice::GetClockCounters(uint64_t *gpu, uint64_t *cpu) {
|
||||
void *priv_data;
|
||||
int priv_size;
|
||||
|
||||
priv_size = rocr_proxy::CreateCalibratedTimestampsPrivData(&priv_data);
|
||||
|
||||
D3DKMT_ESCAPE d3dkmt_escape;
|
||||
memset(&d3dkmt_escape, 0, sizeof(d3dkmt_escape));
|
||||
|
||||
d3dkmt_escape.hAdapter = adapter_;
|
||||
d3dkmt_escape.hDevice = device_;
|
||||
d3dkmt_escape.hContext = 0; //KMD only use device to identify the process
|
||||
d3dkmt_escape.Type = D3DKMT_ESCAPE_DRIVERPRIVATE;
|
||||
d3dkmt_escape.pPrivateDriverData = priv_data;
|
||||
d3dkmt_escape.PrivateDriverDataSize = priv_size;
|
||||
d3dkmt_escape.Flags.HardwareAccess = true;
|
||||
|
||||
NTSTATUS status = D3DKMTEscape(&d3dkmt_escape);
|
||||
if (status) {
|
||||
debug_print("%s status %d \n", __FUNCTION__, status);
|
||||
} else {
|
||||
rocr_proxy::QueryCalibratedTimestamps(priv_data, gpu, cpu);
|
||||
}
|
||||
rocr_proxy::DestroyPrivData(priv_data);
|
||||
}
|
||||
|
||||
bool WDDMDevice::CreateQueue(WDDMQueue *queue) {
|
||||
if (!CreateContext(queue->queue_engine, &queue->context))
|
||||
return false;
|
||||
|
||||
GpuMemoryCreateInfo create_info{};
|
||||
create_info.size = queue->cmdbuf_size;
|
||||
create_info.domain = rocr_proxy::kSystem;
|
||||
|
||||
GpuMemory *gpu_mem = nullptr;
|
||||
auto code = CreateGpuMemory(create_info, &gpu_mem);
|
||||
if (code != ErrorCode::Success)
|
||||
goto err_out0;
|
||||
|
||||
queue->cmdbuf = gpu_mem->GetGpuMemoryHandle();
|
||||
queue->cmdbuf_addr = gpu_mem->GpuAddress();
|
||||
|
||||
if (queue->Init())
|
||||
goto err_out1;
|
||||
|
||||
return true;
|
||||
|
||||
err_out1:
|
||||
delete gpu_mem;
|
||||
err_out0:
|
||||
DestroyContext(queue->context);
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
void WDDMDevice::DestroyQueue(WDDMQueue *queue) {
|
||||
|
||||
queue->Fini();
|
||||
|
||||
auto cmdbuf_mem = GpuMemory::Convert(queue->cmdbuf);
|
||||
delete cmdbuf_mem;
|
||||
|
||||
DestroyContext(queue->context);
|
||||
}
|
||||
|
||||
bool WDDMDevice::SubmitToSwQueue(WDDMQueue *queue, uint64_t command_addr,
|
||||
uint64_t command_size, uint64_t fence_value) {
|
||||
void *priv_data;
|
||||
int priv_size;
|
||||
|
||||
priv_size = rocr_proxy::CreateSubmitPrivData(&priv_data, queue->queue, command_addr, command_size, false);
|
||||
|
||||
D3DKMT_SUBMITCOMMAND args = {0};
|
||||
args.Commands = command_addr;
|
||||
args.CommandLength = command_size;
|
||||
args.BroadcastContextCount = 1;
|
||||
args.BroadcastContext[0] = queue->context;
|
||||
args.pPrivateDriverData = priv_data;
|
||||
args.PrivateDriverDataSize = priv_size;
|
||||
|
||||
NTSTATUS ret = D3DKMTSubmitCommand(&args);
|
||||
if (ret != STATUS_SUCCESS) {
|
||||
fprintf(stderr, "%s fail %x\n", __FUNCTION__, ret);
|
||||
rocr_proxy::DestroyPrivData(priv_data);
|
||||
return false;
|
||||
}
|
||||
|
||||
rocr_proxy::DestroyPrivData(priv_data);
|
||||
|
||||
if (!GpuSignal(queue->context, &queue->syncobj, &fence_value, 1))
|
||||
return false;
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
bool WDDMDevice::CreateHwQueue(WDDMQueue *queue) {
|
||||
void *priv_data;
|
||||
int priv_size;
|
||||
|
||||
bool FwManagedGfxState = SupportStateShadowingByCpFw();
|
||||
priv_size = rocr_proxy::CreateHwQueuePrivData(&priv_data, queue->context,
|
||||
FwManagedGfxState, queue->prio);
|
||||
|
||||
D3DKMT_CREATEHWQUEUE createHwQueue = {0};
|
||||
createHwQueue.hHwContext = queue->context;
|
||||
createHwQueue.Flags.DisableGpuTimeout = rocr_proxy::ShouldDisableGpuTimeout(queue->queue_engine, &device_info_);
|
||||
createHwQueue.pPrivateDriverData = priv_data;
|
||||
createHwQueue.PrivateDriverDataSize = priv_size;
|
||||
|
||||
NTSTATUS ret = D3DKMTCreateHwQueue(&createHwQueue);
|
||||
if (ret != STATUS_SUCCESS) {
|
||||
fprintf(stderr, "%s fail %x\n", __FUNCTION__, ret);
|
||||
rocr_proxy::DestroyPrivData(priv_data);
|
||||
return false;
|
||||
}
|
||||
|
||||
rocr_proxy::DestroyPrivData(priv_data);
|
||||
|
||||
queue->queue = createHwQueue.hHwQueue;
|
||||
queue->syncobj = createHwQueue.hHwQueueProgressFence;
|
||||
queue->sync_addr = (uint64_t *)createHwQueue.HwQueueProgressFenceCPUVirtualAddress;
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
bool WDDMDevice::DestroyHwQueue(WDDMQueue *queue) {
|
||||
D3DKMT_DESTROYHWQUEUE DestroyHwQueue = {
|
||||
.hHwQueue = queue->queue,
|
||||
};
|
||||
|
||||
NTSTATUS ret = D3DKMTDestroyHwQueue(&DestroyHwQueue);
|
||||
if (ret != STATUS_SUCCESS) {
|
||||
fprintf(stderr, "%s fail %x\n", __FUNCTION__, ret);
|
||||
return false;
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
bool WDDMDevice::SubmitToHwQueue(WDDMQueue *queue, uint64_t command_addr,
|
||||
uint64_t command_size, uint64_t fence_value) {
|
||||
void *priv_data;
|
||||
int priv_size;
|
||||
|
||||
priv_size = rocr_proxy::CreateSubmitPrivData(&priv_data, queue->queue, command_addr, command_size, true);
|
||||
|
||||
D3DKMT_SUBMITCOMMANDTOHWQUEUE args = {0};
|
||||
args.hHwQueue = queue->queue;
|
||||
args.HwQueueProgressFenceId = fence_value;
|
||||
args.CommandBuffer = command_addr;
|
||||
args.CommandLength = command_size;
|
||||
args.pPrivateDriverData = priv_data;
|
||||
args.PrivateDriverDataSize = priv_size;
|
||||
|
||||
NTSTATUS ret = D3DKMTSubmitCommandToHwQueue(&args);
|
||||
if (ret != STATUS_SUCCESS) {
|
||||
fprintf(stderr, "%s fail %x\n", __FUNCTION__, ret);
|
||||
rocr_proxy::DestroyPrivData(priv_data);
|
||||
return false;
|
||||
}
|
||||
|
||||
rocr_proxy::DestroyPrivData(priv_data);
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
} // namespace core
|
||||
} // namespace rocr
|
||||
@@ -0,0 +1,467 @@
|
||||
#include <cinttypes>
|
||||
#include <cassert>
|
||||
#include "inc/wddm/gpu_memory.h"
|
||||
#include "inc/wddm/device.h"
|
||||
#include "util/utils.h"
|
||||
|
||||
using namespace std;
|
||||
|
||||
namespace rocr {
|
||||
namespace core {
|
||||
|
||||
size_t GpuMemory::CalcChunkNumbers(gpusize size) {
|
||||
const auto chunk_size = core::WDDMDevice::GpuMemoryChunkSize;
|
||||
return (size + chunk_size - 1) / chunk_size;
|
||||
}
|
||||
|
||||
gpusize GpuMemory::AdjustSize(gpusize size) const {
|
||||
const auto &device_info = device_->DeviceInfo();
|
||||
|
||||
if (device_info.enable_big_page_alignment && desc_.domain == rocr_proxy::kLocal) {
|
||||
uint32_t alignment = device_info.big_page_alignment_size;
|
||||
// BigPage is only supported for allocations > bigPageMinAlignment.
|
||||
// Also, if bigPageMinAlignment == 0, BigPage optimization is not supported per KMD.
|
||||
// We do either LargePage or BigPage alignment, whichever has a higher value.
|
||||
if ((device_info.hw_big_page_min_alignment_size > 0) && (size > device_info.hw_big_page_min_alignment_size)) {
|
||||
alignment = std::max(alignment, device_info.hw_big_page_min_alignment_size);
|
||||
if (size > device_info.hw_big_page_alignment_size)
|
||||
alignment = std::max(alignment, device_info.hw_big_page_alignment_size);
|
||||
}
|
||||
if (alignment > 0)
|
||||
size = AlignUp(size, alignment);
|
||||
} else {
|
||||
const size_t min_size = 4096;
|
||||
size = AlignUp(size, min_size);
|
||||
}
|
||||
return size;
|
||||
}
|
||||
|
||||
GpuMemory::GpuMemory(WDDMDevice *device) : device_(device) {
|
||||
num_allocations_ = 0;
|
||||
alloc_handles_ptr_ = nullptr;
|
||||
alloc_handle_ = 0;
|
||||
resource_ = 0;
|
||||
}
|
||||
|
||||
GpuMemory::~GpuMemory() {
|
||||
FreeGpuVirtualAddress(GpuAddress(), Size());
|
||||
FreePhysicalMemory();
|
||||
}
|
||||
|
||||
ErrorCode GpuMemory::Init(const GpuMemoryCreateInfo &create_info) {
|
||||
desc_.domain = create_info.domain;
|
||||
desc_.adapter_luid = device_->GetLuid();
|
||||
desc_.client_size = create_info.size;
|
||||
desc_.alignment = create_info.alignment;
|
||||
desc_.mem_flags = create_info.mem_flags;
|
||||
desc_.engine_flag = create_info.engine_flag;
|
||||
desc_.flags.is_virtual = create_info.flags.virtual_alloc;
|
||||
desc_.flags.is_physical_only = create_info.flags.physical_only;
|
||||
desc_.flags.is_shared = create_info.flags.interprocess;
|
||||
desc_.flags.is_locked = create_info.flags.locked;
|
||||
|
||||
desc_.size = AdjustSize(desc_.client_size);
|
||||
|
||||
if (IsUserMemory() || IsSystem())
|
||||
desc_.cpu_addr = create_info.user_ptr;
|
||||
|
||||
num_allocations_ = CalcChunkNumbers(Size());
|
||||
if (num_allocations_ == 1)
|
||||
alloc_handles_ptr_ = &alloc_handle_;
|
||||
else
|
||||
alloc_handles_ptr_ = new WinAllocationHandle[num_allocations_];
|
||||
|
||||
memset(alloc_handles_ptr_, 0, num_allocations_ * sizeof(WinAllocationHandle));
|
||||
|
||||
auto code = ErrorCode::Success;
|
||||
|
||||
if (IsPhysicalOnly()) {
|
||||
code = CreatePhysicalMemory();
|
||||
return code;
|
||||
}
|
||||
|
||||
code = ReserveGpuVirtualAddress(create_info.va_hint, Size(), create_info.alignment);
|
||||
if (IsVirtual() || (code != ErrorCode::Success))
|
||||
return code;
|
||||
|
||||
bool physical_created = false;
|
||||
|
||||
auto guard = MakeScopeGuard([this, &physical_created, &code]() {
|
||||
if (code != ErrorCode::Success) {
|
||||
|
||||
if (physical_created) {
|
||||
FreePhysicalMemory();
|
||||
}
|
||||
FreeGpuVirtualAddress(GpuAddress(), Size());
|
||||
}
|
||||
});
|
||||
(void)guard;
|
||||
|
||||
code = CreatePhysicalMemory();
|
||||
if (code != ErrorCode::Success)
|
||||
return code;
|
||||
|
||||
physical_created = true;
|
||||
|
||||
code = MapGpuVirtualAddress(GpuAddress(), Size());
|
||||
if (code != ErrorCode::Success)
|
||||
return code;
|
||||
|
||||
code = MakeResident();
|
||||
if (code != ErrorCode::Success)
|
||||
return code;
|
||||
|
||||
if (!GetDevice()->WaitOnPagingFenceFromCpu())
|
||||
code = ErrorCode::Unknown;
|
||||
|
||||
return code;
|
||||
}
|
||||
|
||||
ErrorCode GpuMemory::UnmapGpuVirtualAddress(const gpusize addr, const gpusize size, gpusize offset) {
|
||||
auto code = ErrorCode::Success;
|
||||
size_t i = 0;
|
||||
auto map_addr = addr;
|
||||
auto map_size = size;
|
||||
|
||||
while (offset >= core::WDDMDevice::GpuMemoryChunkSize) {
|
||||
offset -= core::WDDMDevice::GpuMemoryChunkSize;
|
||||
i += 1;
|
||||
}
|
||||
|
||||
while (map_size > 0) {
|
||||
auto block_size = std::min(map_size, core::WDDMDevice::GpuMemoryChunkSize);
|
||||
|
||||
D3DDDI_MAPGPUVIRTUALADDRESS args{};
|
||||
|
||||
args.hPagingQueue = device_->PagingQueue();
|
||||
args.BaseAddress = map_addr;
|
||||
args.hAllocation = GetAllocationHandle(i);
|
||||
args.SizeInPages = block_size / 0x1000;
|
||||
args.Protection.NoAccess = 1;
|
||||
|
||||
code = thunk::MapGpuVirtualAddress(&args);
|
||||
|
||||
if (code == ErrorCode::NotReady)
|
||||
device_->UpdatePageFence(args.PagingFenceValue);
|
||||
else if (code != ErrorCode::Success)
|
||||
break;
|
||||
|
||||
map_addr += block_size;
|
||||
map_size -= block_size;
|
||||
offset = 0; // reset second unmapped allocation offset to zero
|
||||
i += 1;
|
||||
}
|
||||
|
||||
return code;
|
||||
}
|
||||
|
||||
ErrorCode GpuMemory::MapGpuVirtualAddress(const gpusize addr, const gpusize size, gpusize offset) {
|
||||
|
||||
auto code = ErrorCode::Success;
|
||||
size_t i = 0;
|
||||
auto map_addr = addr;
|
||||
auto map_size = size;
|
||||
const size_t _4K = 0x1000;
|
||||
|
||||
while (offset >= core::WDDMDevice::GpuMemoryChunkSize) {
|
||||
offset -= core::WDDMDevice::GpuMemoryChunkSize;
|
||||
i += 1;
|
||||
}
|
||||
const size_t first_chunk = i;
|
||||
const auto first_chunk_offset = offset;
|
||||
/* Found two limitation for local vram:
|
||||
* 1. invisible vram va has to be 64K aligned, otherwise map gpu va fail
|
||||
* 2. visible vram can not be cpu mapped when command submission or after gpu mapped
|
||||
*/
|
||||
while (map_size > 0) {
|
||||
auto block_size = std::min(map_size, core::WDDMDevice::GpuMemoryChunkSize);
|
||||
|
||||
D3DDDI_MAPGPUVIRTUALADDRESS args{};
|
||||
|
||||
args.hPagingQueue = device_->PagingQueue();
|
||||
args.BaseAddress = map_addr;
|
||||
args.hAllocation = GetAllocationHandle(i);
|
||||
args.OffsetInPages = offset / _4K;
|
||||
args.SizeInPages = block_size / _4K;
|
||||
args.Protection.Write = 1;
|
||||
|
||||
code = thunk::MapGpuVirtualAddress(&args);
|
||||
|
||||
if (code != ErrorCode::Success) {
|
||||
if (code == ErrorCode::NotReady) {
|
||||
const uint64_t fence_value = args.PagingFenceValue;
|
||||
device_->UpdatePageFence(fence_value);
|
||||
code = ErrorCode::Success;
|
||||
} else
|
||||
break;
|
||||
}
|
||||
|
||||
map_addr += block_size;
|
||||
map_size -= block_size;
|
||||
offset = 0; // reset second mapped allocation offset to zero
|
||||
i++;
|
||||
}
|
||||
|
||||
if (code != ErrorCode::Success) {
|
||||
// Map failed, unmap partial mapped block
|
||||
offset = first_chunk_offset;
|
||||
map_addr = addr;
|
||||
map_size = size;
|
||||
for (size_t j = first_chunk; j < i; j++) {
|
||||
auto block_size = std::min(map_size, core::WDDMDevice::GpuMemoryChunkSize);
|
||||
|
||||
D3DDDI_MAPGPUVIRTUALADDRESS args{};
|
||||
|
||||
args.hPagingQueue = device_->PagingQueue();
|
||||
args.BaseAddress = map_addr;
|
||||
args.hAllocation = 0;
|
||||
args.OffsetInPages = offset / _4K;
|
||||
args.SizeInPages = block_size / _4K;
|
||||
args.Protection.NoAccess = 1;
|
||||
|
||||
auto unmap_code = thunk::MapGpuVirtualAddress(&args);
|
||||
if (unmap_code == ErrorCode::NotReady)
|
||||
device_->UpdatePageFence(args.PagingFenceValue);
|
||||
|
||||
map_addr += block_size;
|
||||
map_size -= block_size;
|
||||
}
|
||||
}
|
||||
return code;
|
||||
}
|
||||
|
||||
ErrorCode GpuMemory::ReserveGpuVirtualAddress(gpusize base_virt_addr, gpusize size, gpusize alignment) {
|
||||
gpusize gpu_virt_addr = 0;
|
||||
auto status = device_->ReserveGpuVirtualAddress(desc_.domain, base_virt_addr, size, &gpu_virt_addr, alignment,
|
||||
desc_.flags.is_locked);
|
||||
if (status == ErrorCode::Success) {
|
||||
desc_.gpu_addr = gpu_virt_addr;
|
||||
|
||||
if (IsSystem())
|
||||
desc_.cpu_addr = reinterpret_cast<void *>(desc_.gpu_addr);
|
||||
}
|
||||
return status;
|
||||
}
|
||||
|
||||
ErrorCode GpuMemory::FreeGpuVirtualAddress(gpusize base_addr, gpusize size) {
|
||||
return base_addr != 0 ?
|
||||
device_->FreeGpuVirtualAddress(desc_.domain, base_addr, size) :
|
||||
ErrorCode::Success;
|
||||
}
|
||||
|
||||
ErrorCode GpuMemory::CreatePhysicalMemory() {
|
||||
|
||||
assert(!IsVirtual() && NumChunks() > 0);
|
||||
|
||||
const auto num_allocations = NumChunks();
|
||||
void *priv_drv_data;
|
||||
void *alloc_priv;
|
||||
int priv_drv_data_size;
|
||||
int alloc_priv_data_size;
|
||||
|
||||
if (!rocr_proxy::CreatePrivateAllocInfo(NumChunks(), &priv_drv_data, &alloc_priv,
|
||||
&priv_drv_data_size, &alloc_priv_data_size))
|
||||
return ErrorCode::OutOfMemory;
|
||||
|
||||
auto alloc_info = reinterpret_cast<D3DDDI_ALLOCATIONINFO2*>(
|
||||
static_cast<unsigned char*>(priv_drv_data) + priv_drv_data_size * num_allocations);
|
||||
|
||||
size_t size = desc_.size;
|
||||
uint64_t addr = desc_.gpu_addr;
|
||||
char *cpu_addr = static_cast<char *>(desc_.cpu_addr);
|
||||
const auto &device_info = GetDevice()->DeviceInfo();
|
||||
|
||||
for (size_t i = 0; i < num_allocations; i++) {
|
||||
|
||||
void* priv_data = (void*)((char*)priv_drv_data + priv_drv_data_size * i);
|
||||
size_t block_size = std::min(size, core::WDDMDevice::GpuMemoryChunkSize);
|
||||
|
||||
if (IsUserMemory() || IsSystem()) {
|
||||
rocr_proxy::SetAllocationInfo(priv_data, block_size, desc_.domain, 0, desc_.mem_flags, desc_.engine_flag, device_info);
|
||||
alloc_info[i].pSystemMem = static_cast<void *>(cpu_addr);
|
||||
cpu_addr += block_size;
|
||||
} else {
|
||||
rocr_proxy::SetAllocationInfo(priv_data, block_size, desc_.domain, addr, desc_.mem_flags, desc_.engine_flag, device_info);
|
||||
}
|
||||
|
||||
size -= block_size;
|
||||
addr += block_size;
|
||||
|
||||
alloc_info[i].pPrivateDriverData = priv_data;
|
||||
alloc_info[i].PrivateDriverDataSize = priv_drv_data_size;
|
||||
alloc_info[i].VidPnSourceId = D3DDDI_ID_UNINITIALIZED;
|
||||
}
|
||||
|
||||
D3DKMT_CREATEALLOCATION args = {};
|
||||
args.hDevice = device_->DeviceHandle();
|
||||
args.pPrivateDriverData = alloc_priv;
|
||||
args.PrivateDriverDataSize = alloc_priv_data_size;
|
||||
args.NumAllocations = num_allocations;
|
||||
args.pAllocationInfo2 = alloc_info;
|
||||
|
||||
SharedHandleInfo shared_info;
|
||||
if (IsShared()) {
|
||||
shared_info.size = desc_.size;
|
||||
shared_info.client_size = desc_.client_size;
|
||||
shared_info.domain = desc_.domain;
|
||||
shared_info.adapter_luid = desc_.adapter_luid;
|
||||
shared_info.flags = reinterpret_cast<uint32_t>(desc_.flags.reserved);
|
||||
shared_info.mem_flags = desc_.mem_flags;
|
||||
args.pPrivateRuntimeData = &shared_info;
|
||||
args.PrivateRuntimeDataSize = sizeof(shared_info);
|
||||
args.Flags.NtSecuritySharing = 1;
|
||||
args.Flags.CreateShared = 1;
|
||||
args.Flags.CreateResource = 1;
|
||||
}
|
||||
|
||||
auto status = thunk::CreateAllocation(&args);
|
||||
if (status == ErrorCode::Success) {
|
||||
for (size_t i = 0; i < num_allocations; i++)
|
||||
alloc_handles_ptr_[i] = alloc_info[i].hAllocation;
|
||||
|
||||
resource_ = args.hResource;
|
||||
}
|
||||
rocr_proxy::DestroyPrivateAllocInfo(priv_drv_data, alloc_priv);
|
||||
return status;
|
||||
}
|
||||
|
||||
ErrorCode GpuMemory::FreePhysicalMemory() {
|
||||
auto code = ErrorCode::Success;
|
||||
|
||||
if (alloc_handles_ptr_ == nullptr || (NumChunks() == 1 && *alloc_handles_ptr_ == 0))
|
||||
return code;
|
||||
|
||||
code = thunk::DestroyAllocation(device_->DeviceHandle(),
|
||||
resource_,
|
||||
NumChunks(),
|
||||
alloc_handles_ptr_);
|
||||
if (NumChunks() > 1)
|
||||
delete[] alloc_handles_ptr_;
|
||||
|
||||
alloc_handles_ptr_ = nullptr;
|
||||
return code;
|
||||
}
|
||||
|
||||
ErrorCode GpuMemory::MakeResident() {
|
||||
|
||||
D3DDDI_MAKERESIDENT args = {};
|
||||
args.hPagingQueue = device_->PagingQueue();
|
||||
args.NumAllocations = NumChunks();
|
||||
args.AllocationList = alloc_handles_ptr_;
|
||||
args.Flags.CantTrimFurther = 1;
|
||||
|
||||
auto code = thunk::MakeResident(&args);
|
||||
if (code == ErrorCode::NotReady) {
|
||||
const auto fence_value = args.PagingFenceValue;
|
||||
device_->UpdatePageFence(fence_value);
|
||||
code = ErrorCode::Success;
|
||||
}
|
||||
return code;
|
||||
}
|
||||
|
||||
ErrorCode GpuMemory::Evict() {
|
||||
|
||||
D3DKMT_EVICT args = {};
|
||||
args.hDevice = device_->DeviceHandle();
|
||||
args.NumAllocations = NumChunks();
|
||||
args.AllocationList = alloc_handles_ptr_;
|
||||
|
||||
return thunk::Evict(&args);
|
||||
}
|
||||
|
||||
ErrorCode GpuMemory::ExportPhysicalHandle(int* dmabuf_fd, uint32_t flags) {
|
||||
if (IsShared())
|
||||
return thunk::ShareObjects(num_allocations_, resource_, flags, dmabuf_fd);
|
||||
else
|
||||
return ErrorCode::UnSupported;
|
||||
}
|
||||
|
||||
|
||||
ErrorCode GpuMemory::ImportPhysicalHandle(int dmabuf_fd) {
|
||||
D3DKMT_QUERYRESOURCEINFOFROMNTHANDLE query_args;
|
||||
|
||||
if (dmabuf_fd <= 0)
|
||||
return ErrorCode::InvalidateParams;
|
||||
|
||||
memset(&query_args, 0, sizeof(query_args));
|
||||
query_args.hDevice = device_->DeviceHandle();
|
||||
query_args.hNtHandle = reinterpret_cast<HANDLE>(dmabuf_fd);
|
||||
auto ret = thunk::QueryResourceInfoFromNtHandle(&query_args);
|
||||
if (ret != ErrorCode::Success) {
|
||||
debug_print("%s query resource info from nt handle failed %d\n", __FUNCTION__, static_cast<int>(ret));
|
||||
return ErrorCode::InvalidateParams;
|
||||
}
|
||||
debug_print("wsl-rocr: import from nt handle %d, get allocation number %d,"
|
||||
" runtime data size %" PRIx64 " total driver data size %" PRIx64 "resource data size=%" PRIx64 "\n",
|
||||
dmabuf_fd,
|
||||
query_args.NumAllocations,
|
||||
query_args.PrivateRuntimeDataSize,
|
||||
query_args.TotalPrivateDriverDataSize,
|
||||
query_args.ResourcePrivateDriverDataSize);
|
||||
|
||||
SharedHandleInfo shared_info;
|
||||
if(sizeof(shared_info) != query_args.PrivateRuntimeDataSize) {
|
||||
debug_print("%s shared hanle info size mismatch:%d vs %ld\n",
|
||||
__FUNCTION__, query_args.PrivateRuntimeDataSize, sizeof(shared_info));
|
||||
return ErrorCode::UnSupported;
|
||||
}
|
||||
|
||||
uint32_t total_size = query_args.NumAllocations * sizeof(D3DDDI_OPENALLOCATIONINFO2) +
|
||||
query_args.TotalPrivateDriverDataSize +
|
||||
query_args.ResourcePrivateDriverDataSize;
|
||||
D3DDDI_OPENALLOCATIONINFO2 *open_info =
|
||||
reinterpret_cast<D3DDDI_OPENALLOCATIONINFO2*> (calloc(1, total_size));
|
||||
if (!open_info) {
|
||||
debug_print("%s alloc open_info failed, NumAllocations:%d\n",
|
||||
__FUNCTION__, query_args.NumAllocations);
|
||||
return ErrorCode::OutOfMemory;
|
||||
}
|
||||
|
||||
alloc_handles_ptr_ = new WinAllocationHandle[query_args.NumAllocations];
|
||||
|
||||
D3DKMT_OPENRESOURCEFROMNTHANDLE open_args;
|
||||
memset(&open_args, 0, sizeof(open_args));
|
||||
open_args.hDevice = query_args.hDevice;
|
||||
open_args.hNtHandle = query_args.hNtHandle;
|
||||
open_args.NumAllocations = query_args.NumAllocations;
|
||||
open_args.pOpenAllocationInfo2 = open_info;
|
||||
open_args.TotalPrivateDriverDataBufferSize = query_args.TotalPrivateDriverDataSize;
|
||||
open_args.pTotalPrivateDriverDataBuffer = reinterpret_cast<void*>
|
||||
(open_args.pOpenAllocationInfo2 + open_args.NumAllocations);
|
||||
open_args.ResourcePrivateDriverDataSize = query_args.ResourcePrivateDriverDataSize;
|
||||
open_args.pResourcePrivateDriverData = reinterpret_cast<void*>
|
||||
(((uint64_t)open_args.pTotalPrivateDriverDataBuffer) +
|
||||
open_args.TotalPrivateDriverDataBufferSize);
|
||||
open_args.PrivateRuntimeDataSize = query_args.PrivateRuntimeDataSize;
|
||||
open_args.pPrivateRuntimeData = reinterpret_cast<void*> (&shared_info);
|
||||
|
||||
ret = thunk::OpenResourceFromNtHandle(&open_args);
|
||||
if (ret != ErrorCode::Success) {
|
||||
ret = ErrorCode::InvalidateParams;
|
||||
debug_print("%s open resource failed %d\n", __FUNCTION__, static_cast<int>(ret));
|
||||
goto err_out;
|
||||
}
|
||||
|
||||
desc_.size = shared_info.size;
|
||||
desc_.client_size = shared_info.client_size;
|
||||
desc_.domain = shared_info.domain;
|
||||
desc_.flags.reserved = shared_info.flags;
|
||||
desc_.mem_flags = shared_info.mem_flags;
|
||||
desc_.adapter_luid = shared_info.adapter_luid;
|
||||
resource_ = open_args.hResource;
|
||||
num_allocations_ = open_args.NumAllocations;
|
||||
for (int i = 0; i < num_allocations_; i++)
|
||||
alloc_handles_ptr_[i] = open_info[i].hAllocation;
|
||||
|
||||
free(open_info);
|
||||
return ErrorCode::Success;
|
||||
|
||||
err_out:
|
||||
delete[] alloc_handles_ptr_;
|
||||
alloc_handles_ptr_ = nullptr;
|
||||
free(open_info);
|
||||
return ret;
|
||||
}
|
||||
|
||||
} // namespace code
|
||||
} // namespace rocr
|
||||
@@ -0,0 +1,989 @@
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
//
|
||||
// The University of Illinois/NCSA
|
||||
// Open Source License (NCSA)
|
||||
//
|
||||
// Copyright (c) 2020, Advanced Micro Devices, Inc. All rights reserved.
|
||||
//
|
||||
// Developed by:
|
||||
//
|
||||
// AMD Research and AMD HSA Software Development
|
||||
//
|
||||
// Advanced Micro Devices, Inc.
|
||||
//
|
||||
// www.amd.com
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
// of this software and associated documentation files (the "Software"), to
|
||||
// deal with the Software without restriction, including without limitation
|
||||
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
// and/or sell copies of the Software, and to permit persons to whom the
|
||||
// Software is furnished to do so, subject to the following conditions:
|
||||
//
|
||||
// - Redistributions of source code must retain the above copyright notice,
|
||||
// this list of conditions and the following disclaimers.
|
||||
// - Redistributions in binary form must reproduce the above copyright
|
||||
// notice, this list of conditions and the following disclaimers in
|
||||
// the documentation and/or other materials provided with the distribution.
|
||||
// - Neither the names of Advanced Micro Devices, Inc,
|
||||
// nor the names of its contributors may be used to endorse or promote
|
||||
// products derived from this Software without specific prior written
|
||||
// permission.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
|
||||
// THE CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
|
||||
// OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
|
||||
// ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
||||
// DEALINGS WITH THE SOFTWARE.
|
||||
//
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
#include <cstring>
|
||||
#include <cinttypes>
|
||||
#include <cstddef>
|
||||
|
||||
#include "inc/wddm/queue.h"
|
||||
#include "inc/registers.h"
|
||||
#include "libhsakmt.h"
|
||||
|
||||
namespace rocr {
|
||||
namespace core {
|
||||
|
||||
hsa_status_t WDDMQueue::SwsInit(void) {
|
||||
if (!device->CreateSyncobj(&syncobj, &sync_addr))
|
||||
return HSA_STATUS_ERROR;
|
||||
|
||||
if (device->AllocUserQueueMemFromUMD()) {
|
||||
|
||||
GpuMemory *gpu_mem = nullptr;
|
||||
GpuMemoryCreateInfo create_info{};
|
||||
|
||||
create_info.domain = rocr_proxy::kUserQueue;
|
||||
create_info.size = device->GetSwsQueueSize();
|
||||
create_info.engine_flag = rocr_proxy::QueueEngine2EngineFlag(queue_engine);
|
||||
|
||||
auto code = device->CreateGpuMemory(create_info, &gpu_mem);
|
||||
if (code != ErrorCode::Success) {
|
||||
device->DestroySyncobj(syncobj);
|
||||
return HSA_STATUS_ERROR;
|
||||
}
|
||||
|
||||
queue_mem = gpu_mem->GetGpuMemoryHandle();
|
||||
queue = gpu_mem->GetAllocationHandle(0);
|
||||
}
|
||||
|
||||
return HSA_STATUS_SUCCESS;
|
||||
}
|
||||
|
||||
hsa_status_t WDDMQueue::SwsFini(void) {
|
||||
device->DestroySyncobj(syncobj);
|
||||
|
||||
return HSA_STATUS_SUCCESS;
|
||||
}
|
||||
|
||||
hsa_status_t WDDMQueue::SwsSubmit(uint64_t command_addr,
|
||||
uint64_t command_size,
|
||||
uint64_t fence_value) {
|
||||
if (!device->SubmitToSwQueue(this, command_addr, command_size, fence_value))
|
||||
return HSA_STATUS_ERROR;
|
||||
|
||||
return HSA_STATUS_SUCCESS;
|
||||
}
|
||||
|
||||
hsa_status_t WDDMQueue::HwsInit(void) {
|
||||
if (!device->CreateHwQueue(this))
|
||||
return HSA_STATUS_ERROR;
|
||||
|
||||
return HSA_STATUS_SUCCESS;
|
||||
}
|
||||
|
||||
hsa_status_t WDDMQueue::HwsFini(void) {
|
||||
if (!device->DestroyHwQueue(this))
|
||||
return HSA_STATUS_ERROR;
|
||||
|
||||
return HSA_STATUS_SUCCESS;
|
||||
}
|
||||
|
||||
hsa_status_t WDDMQueue::HwsSubmit(uint64_t command_addr,
|
||||
uint64_t command_size,
|
||||
uint64_t fence_value) {
|
||||
if (!device->SubmitToHwQueue(this, command_addr, command_size, fence_value))
|
||||
return HSA_STATUS_ERROR;
|
||||
|
||||
return HSA_STATUS_SUCCESS;
|
||||
}
|
||||
|
||||
hsa_status_t WDDMQueue::SetPriority(hsa_amd_queue_priority_t priority) {
|
||||
if (!use_hws)
|
||||
return HSA_STATUS_SUCCESS;
|
||||
|
||||
rocr_proxy::SchedLevel new_prio = ConvertSchedLevel(priority);
|
||||
if (prio == new_prio)
|
||||
return HSA_STATUS_SUCCESS;
|
||||
|
||||
debug_print("set prio %d -> %d\n", prio, new_prio);
|
||||
device->DestroyHwQueue(this);
|
||||
|
||||
prio = new_prio;
|
||||
return HwsInit();
|
||||
}
|
||||
|
||||
extern "C" void (*fn_hsa_signal_store_screlease)(hsa_signal_t hsa_signal, hsa_signal_value_t value);
|
||||
void ComputeQueue::HandleError(hsa_status_t status) {
|
||||
hsa_signal_t sig = amd_queue_rocr_->queue_inactive_signal;
|
||||
hsa_signal_value_t val = -1;
|
||||
|
||||
struct queue_error_t {
|
||||
uint32_t code;
|
||||
hsa_status_t status;
|
||||
};
|
||||
static const queue_error_t QueueErrors[] = {
|
||||
{2, HSA_STATUS_ERROR_INCOMPATIBLE_ARGUMENTS},
|
||||
{4, HSA_STATUS_ERROR_INVALID_ALLOCATION},
|
||||
{8, HSA_STATUS_ERROR_INVALID_CODE_OBJECT},
|
||||
//{16, HSA_STATUS_ERROR_INVALID_ARGUMENT},
|
||||
{32, HSA_STATUS_ERROR_INVALID_PACKET_FORMAT},
|
||||
{64, HSA_STATUS_ERROR_INVALID_ARGUMENT},
|
||||
//{128, HSA_STATUS_ERROR_OUT_OF_REGISTERS},
|
||||
//{0x20000000, HSA_STATUS_ERROR_MEMORY_APERTURE_VIOLATION},
|
||||
//{0x40000000, HSA_STATUS_ERROR_ILLEGAL_INSTRUCTION},
|
||||
{0x80000000, HSA_STATUS_ERROR_EXCEPTION},
|
||||
};
|
||||
for (std::size_t i = 0; i < sizeof(QueueErrors) / sizeof(QueueErrors[0]); ++i) {
|
||||
if (QueueErrors[i].status == status) {
|
||||
val = QueueErrors[i].code;
|
||||
debug_print("error %d, sig_val %d\n", status, val);
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (sig.handle) {
|
||||
fn_hsa_signal_store_screlease(sig, val);
|
||||
}
|
||||
if (error_code_) {
|
||||
error_code_->store(val, std::memory_order_release);
|
||||
}
|
||||
}
|
||||
|
||||
void ComputeQueue::AqlToPm4Thread(ComputeQueue *queue) {
|
||||
|
||||
ComputeQueue *cq = queue;
|
||||
// This timing system is used for sleeping this Thread
|
||||
// when one packet is invalid for about 2 seconds.
|
||||
std::chrono::steady_clock::time_point start_time, time;
|
||||
// Set the polling timeout value for 2 seconds
|
||||
const std::chrono::milliseconds kMaxElapsed(2000);
|
||||
uint64_t current_position = cq->GetAqlWriteIndex();
|
||||
bool sleep = false;
|
||||
start_time = std::chrono::steady_clock::now();
|
||||
|
||||
while (true) {
|
||||
if (!cq->IsInvalidPacket()) {
|
||||
hsa_status_t status = cq->Process();
|
||||
if (status != HSA_STATUS_SUCCESS) {
|
||||
fprintf(stderr, "process compute queue fail status = %08x\n", status);
|
||||
queue->HandleError(status);
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
std::unique_lock<std::mutex> lock(queue->thread_cond_lock_);
|
||||
if (current_position == cq->GetAqlWriteIndex()) {
|
||||
time = std::chrono::steady_clock::now();
|
||||
if (time - start_time > kMaxElapsed)
|
||||
sleep = true;
|
||||
} else {
|
||||
start_time = std::chrono::steady_clock::now();
|
||||
current_position = cq->GetAqlWriteIndex();
|
||||
sleep = false;
|
||||
}
|
||||
// CPU wait for valid packet
|
||||
if (cq->GetRingWptr()->load() <= cq->GetRingRptr()->load() ||
|
||||
(sleep && cq->IsInvalidPacket())) {
|
||||
if (queue->thread_stop_)
|
||||
break;
|
||||
debug_print("wait %p wptr=%" PRIx64 " rptr=%" PRIx64 "\n",
|
||||
queue->ring, cq->GetRingWptr()->load(), cq->GetRingRptr()->load());
|
||||
queue->thread_cond_.wait(lock);
|
||||
}
|
||||
}
|
||||
|
||||
debug_print("aql to pm4 thread %p exit\n", queue->ring);
|
||||
}
|
||||
|
||||
ComputeQueue::ComputeQueue(WDDMDevice *device,
|
||||
void *ring,
|
||||
uint64_t ring_size,
|
||||
std::atomic<uint64_t> *ring_wptr,
|
||||
std::atomic<uint64_t> *ring_rptr,
|
||||
volatile int64_t *error_addr,
|
||||
uint32_t cmdbuf_size,
|
||||
uint32_t engine,
|
||||
bool use_hws) :
|
||||
WDDMQueue(device, cmdbuf_size, engine, use_hws),
|
||||
ring(ring),
|
||||
ring_size(ring_size),
|
||||
ring_wptr(ring_wptr),
|
||||
ring_rptr(ring_rptr),
|
||||
error_code_(reinterpret_cast<volatile std::atomic<long int>*>(error_addr)),
|
||||
ib_start_addr(0),
|
||||
ib_size(0),
|
||||
sync_point(0),
|
||||
cmdbuf_aql_frame_write_index(0),
|
||||
cmdbuf_aql_frame_size(0),
|
||||
needs_barrier(true),
|
||||
ready_to_submit(false),
|
||||
thread_stop_(false),
|
||||
scratch_waves_(device->MaxScratchSlotsPerCu() * device->ComputeUnitCount()),
|
||||
scratch_size_per_wave_(0),
|
||||
scratch_size_(0),
|
||||
scratch_base_(nullptr) {
|
||||
|
||||
bool ret = device->CreateQueue(this);
|
||||
assert(ret);
|
||||
|
||||
GpuMemoryCreateInfo create_info{};
|
||||
create_info.size = PAGE_SIZE;
|
||||
create_info.domain = rocr_proxy::kSystem;
|
||||
GpuMemory *gpu_mem = nullptr;
|
||||
auto code = device->CreateGpuMemory(create_info, &gpu_mem);
|
||||
assert(code == ErrorCode::Success);
|
||||
amd_queue_mem_ = gpu_mem->GetGpuMemoryHandle();
|
||||
amd_queue_ = reinterpret_cast<amd_queue_t*>(gpu_mem->GpuAddress());
|
||||
|
||||
aql_to_pm4_thread_ = std::thread(AqlToPm4Thread, this);
|
||||
amd_queue_rocr_ = (amd_queue_t*)((char*)ring_rptr - offsetof(amd_queue_t, read_dispatch_id));
|
||||
}
|
||||
|
||||
ComputeQueue::~ComputeQueue() {
|
||||
thread_cond_lock_.lock();
|
||||
thread_stop_ = true;
|
||||
thread_cond_lock_.unlock();
|
||||
thread_cond_.notify_one();
|
||||
aql_to_pm4_thread_.join();
|
||||
|
||||
//doorbell_signal_->Release();
|
||||
|
||||
device->DestroyQueue(this);
|
||||
|
||||
if (scratch_base_) {
|
||||
auto scratch_gpu_mem = GpuMemory::Convert(scratch_mem_);
|
||||
delete scratch_gpu_mem;
|
||||
}
|
||||
|
||||
auto amd_queue_gpu_mem = GpuMemory::Convert(amd_queue_mem_);
|
||||
delete amd_queue_gpu_mem;
|
||||
}
|
||||
|
||||
void ComputeQueue::InitScratchSRD() {
|
||||
// Populate scratch resource descriptor
|
||||
SQ_BUF_RSRC_WORD0 srd0;
|
||||
|
||||
uintptr_t scratch_base = uintptr_t(scratch_base_);
|
||||
srd0.bits.BASE_ADDRESS = scratch_base;
|
||||
|
||||
uint32_t srd1_u32;
|
||||
|
||||
if (device->Major() < 11) {
|
||||
SQ_BUF_RSRC_WORD1 srd1;
|
||||
|
||||
srd1.bits.BASE_ADDRESS_HI = scratch_base >> 32;
|
||||
srd1.bits.STRIDE = 0;
|
||||
srd1.bits.CACHE_SWIZZLE = 0;
|
||||
srd1.bits.SWIZZLE_ENABLE = 1;
|
||||
|
||||
srd1_u32 = srd1.u32All;
|
||||
} else {
|
||||
SQ_BUF_RSRC_WORD1_GFX11 srd1;
|
||||
|
||||
srd1.bits.BASE_ADDRESS_HI = scratch_base >> 32;
|
||||
srd1.bits.STRIDE = 0;
|
||||
srd1.bits.SWIZZLE_ENABLE = 1;
|
||||
|
||||
srd1_u32 = srd1.u32All;
|
||||
}
|
||||
|
||||
SQ_BUF_RSRC_WORD2 srd2;
|
||||
|
||||
srd2.bits.NUM_RECORDS = scratch_size_;
|
||||
|
||||
uint32_t srd3_u32;
|
||||
|
||||
if (device->Major() < 10) {
|
||||
SQ_BUF_RSRC_WORD3 srd3;
|
||||
|
||||
srd3.bits.DST_SEL_X = SQ_SEL_X;
|
||||
srd3.bits.DST_SEL_Y = SQ_SEL_Y;
|
||||
srd3.bits.DST_SEL_Z = SQ_SEL_Z;
|
||||
srd3.bits.DST_SEL_W = SQ_SEL_W;
|
||||
srd3.bits.NUM_FORMAT = BUF_NUM_FORMAT_UINT;
|
||||
srd3.bits.DATA_FORMAT = BUF_DATA_FORMAT_32;
|
||||
srd3.bits.ELEMENT_SIZE = 1; // 4
|
||||
srd3.bits.INDEX_STRIDE = 3; // 64
|
||||
srd3.bits.ADD_TID_ENABLE = 1;
|
||||
srd3.bits.ATC__CI__VI = 0;
|
||||
srd3.bits.HASH_ENABLE = 0;
|
||||
srd3.bits.HEAP = 0;
|
||||
srd3.bits.MTYPE__CI__VI = 0;
|
||||
srd3.bits.TYPE = SQ_RSRC_BUF;
|
||||
|
||||
srd3_u32 = srd3.u32All;
|
||||
} else if (device->Major() == 10) {
|
||||
SQ_BUF_RSRC_WORD3_GFX10 srd3;
|
||||
|
||||
srd3.bits.DST_SEL_X = SQ_SEL_X;
|
||||
srd3.bits.DST_SEL_Y = SQ_SEL_Y;
|
||||
srd3.bits.DST_SEL_Z = SQ_SEL_Z;
|
||||
srd3.bits.DST_SEL_W = SQ_SEL_W;
|
||||
srd3.bits.FORMAT = BUF_FORMAT_32_UINT;
|
||||
srd3.bits.RESERVED1 = 0;
|
||||
srd3.bits.INDEX_STRIDE = 0; // filled in by CP
|
||||
srd3.bits.ADD_TID_ENABLE = 1;
|
||||
srd3.bits.RESOURCE_LEVEL = 1;
|
||||
srd3.bits.RESERVED2 = 0;
|
||||
srd3.bits.OOB_SELECT = 2; // no bounds check in swizzle mode
|
||||
srd3.bits.TYPE = SQ_RSRC_BUF;
|
||||
|
||||
srd3_u32 = srd3.u32All;
|
||||
} else if (device->Major() == 11) {
|
||||
SQ_BUF_RSRC_WORD3_GFX11 srd3;
|
||||
|
||||
srd3.bits.DST_SEL_X = SQ_SEL_X;
|
||||
srd3.bits.DST_SEL_Y = SQ_SEL_Y;
|
||||
srd3.bits.DST_SEL_Z = SQ_SEL_Z;
|
||||
srd3.bits.DST_SEL_W = SQ_SEL_W;
|
||||
srd3.bits.FORMAT = BUF_FORMAT_32_UINT;
|
||||
srd3.bits.RESERVED1 = 0;
|
||||
srd3.bits.INDEX_STRIDE = 0; // filled in by CP
|
||||
srd3.bits.ADD_TID_ENABLE = 1;
|
||||
srd3.bits.RESERVED2 = 0;
|
||||
srd3.bits.OOB_SELECT = 2; // no bounds check in swizzle mode
|
||||
srd3.bits.TYPE = SQ_RSRC_BUF;
|
||||
|
||||
srd3_u32 = srd3.u32All;
|
||||
} else {
|
||||
SQ_BUF_RSRC_WORD3_GFX12 srd3;
|
||||
srd3.bits.DST_SEL_X = SQ_SEL_X;
|
||||
srd3.bits.DST_SEL_Y = SQ_SEL_Y;
|
||||
srd3.bits.DST_SEL_Z = SQ_SEL_Z;
|
||||
srd3.bits.DST_SEL_W = SQ_SEL_W;
|
||||
srd3.bits.FORMAT = BUF_FORMAT_32_UINT;
|
||||
srd3.bits.RESERVED1 = 0;
|
||||
srd3.bits.INDEX_STRIDE = 0; // filled in by CP
|
||||
srd3.bits.ADD_TID_ENABLE = 1;
|
||||
srd3.bits.WRITE_COMPRESS_ENABLE = 0;
|
||||
srd3.bits.COMPRESSION_EN = 0;
|
||||
srd3.bits.COMPRESSION_ACCESS_MODE = 0;
|
||||
srd3.bits.OOB_SELECT = 2; // no bounds check in swizzle mode
|
||||
srd3.bits.TYPE = SQ_RSRC_BUF;
|
||||
|
||||
srd3_u32 = srd3.u32All;
|
||||
}
|
||||
|
||||
// Update Queue's Scratch descriptor's property
|
||||
amd_queue_->scratch_resource_descriptor[0] = srd0.u32All;
|
||||
amd_queue_->scratch_resource_descriptor[1] = srd1_u32;
|
||||
amd_queue_->scratch_resource_descriptor[2] = srd2.u32All;
|
||||
amd_queue_->scratch_resource_descriptor[3] = srd3_u32;
|
||||
|
||||
// Populate flat scratch parameters in amd_queue_.
|
||||
amd_queue_->scratch_backing_memory_location = scratch_base;
|
||||
amd_queue_->scratch_backing_memory_byte_size = scratch_size_;
|
||||
|
||||
// For backwards compatibility this field records the per-lane scratch
|
||||
// for a 64 lane wavefront. If scratch was allocated for 32 lane waves
|
||||
// then the effective size for a 64 lane wave is halved.
|
||||
amd_queue_->scratch_wave64_lane_byte_size = scratch_size_per_wave_ / 64;
|
||||
|
||||
if (device->Major() < 11) {
|
||||
COMPUTE_TMPRING_SIZE tmpring_size;
|
||||
tmpring_size.bits.WAVESIZE = scratch_size_per_wave_ / 1024;
|
||||
tmpring_size.bits.WAVES = scratch_waves_;
|
||||
|
||||
amd_queue_->compute_tmpring_size = tmpring_size.u32All;
|
||||
} else if (device->Major() == 11) {
|
||||
COMPUTE_TMPRING_SIZE_GFX11 tmpring_size;
|
||||
tmpring_size.bits.WAVESIZE = scratch_size_per_wave_ >> 8;
|
||||
tmpring_size.bits.WAVES = scratch_waves_ / device->NumShaderEngine();
|
||||
|
||||
amd_queue_->compute_tmpring_size = tmpring_size.u32All;
|
||||
} else {
|
||||
COMPUTE_TMPRING_SIZE_GFX12 tmpring_size = {};
|
||||
tmpring_size.bits.WAVESIZE = scratch_size_per_wave_ >> 8;
|
||||
tmpring_size.bits.WAVES = scratch_waves_ / device->NumShaderEngine();
|
||||
|
||||
amd_queue_->compute_tmpring_size = tmpring_size.u32All;
|
||||
}
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
bool ComputeQueue::UpdateScratch(uint32_t private_segment_size, bool wave32) {
|
||||
const uint32_t wavefront = wave32 ? 32 : 64;
|
||||
const uint32_t alignment = 1024 / wavefront;
|
||||
private_segment_size = AlignUp(private_segment_size, alignment);
|
||||
|
||||
uint32_t scratch_size_per_wave = private_segment_size * wavefront;
|
||||
uint32_t scratch_size = scratch_size_per_wave * scratch_waves_;
|
||||
|
||||
if (scratch_size_ >= scratch_size)
|
||||
return true;
|
||||
|
||||
debug_print("need realloc scratch buffer, size %x -> %x\n",
|
||||
scratch_size_, scratch_size);
|
||||
|
||||
GpuMemoryCreateInfo create_info{};
|
||||
create_info.size = scratch_size;
|
||||
create_info.domain = rocr_proxy::kLocal;
|
||||
GpuMemory *gpu_mem = nullptr;
|
||||
auto code = device->CreateGpuMemory(create_info, &gpu_mem);
|
||||
if (code != ErrorCode::Success)
|
||||
return false;
|
||||
|
||||
if (scratch_base_) {
|
||||
auto scratch_gpu_mem = GpuMemory::Convert(scratch_mem_);
|
||||
delete scratch_gpu_mem;
|
||||
}
|
||||
|
||||
scratch_size_per_wave_ = scratch_size_per_wave;
|
||||
scratch_size_ = scratch_size;
|
||||
scratch_base_ = reinterpret_cast<void *>(gpu_mem->GpuAddress());
|
||||
scratch_mem_ = gpu_mem->GetGpuMemoryHandle();
|
||||
|
||||
InitScratchSRD();
|
||||
return true;
|
||||
}
|
||||
|
||||
bool ComputeQueue::RelocateCmdbufScratchBase(uint64_t addr) {
|
||||
if (scratch_base_offset_array_.empty())
|
||||
return true;
|
||||
|
||||
for (size_t i = 0; i < scratch_base_offset_array_.size(); i++) {
|
||||
uint32_t *p_compute_user_data =
|
||||
reinterpret_cast<uint32_t *>(addr + scratch_base_offset_array_[i]);
|
||||
if (device->Major() >= 11) {
|
||||
p_compute_user_data[0] = Ptr48Low32(scratch_base_);
|
||||
p_compute_user_data[1] = Ptr48High8(scratch_base_);
|
||||
} else {
|
||||
p_compute_user_data[0] = PtrLow32(scratch_base_);
|
||||
p_compute_user_data[1] = (p_compute_user_data[1] & 0xffff0000) | PtrHigh32(scratch_base_);
|
||||
}
|
||||
}
|
||||
scratch_base_offset_array_.clear();
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
uint32_t ComputeQueue::UpdateIndexStride(uint32_t srd, bool wave32) {
|
||||
|
||||
assert(device->Major() < 13);
|
||||
|
||||
if (device->Major() == 10) {
|
||||
SQ_BUF_RSRC_WORD3_GFX10 srd3;
|
||||
|
||||
srd3.u32All = srd;
|
||||
srd3.bits.INDEX_STRIDE = wave32 ? 2 : 3;
|
||||
|
||||
return srd3.u32All;
|
||||
} else if (device->Major() == 11) {
|
||||
SQ_BUF_RSRC_WORD3_GFX11 srd3;
|
||||
|
||||
srd3.u32All = srd;
|
||||
srd3.bits.INDEX_STRIDE = wave32 ? 2 : 3;
|
||||
|
||||
return srd3.u32All;
|
||||
} else if (device->Major() == 12) {
|
||||
SQ_BUF_RSRC_WORD3_GFX12 srd3;
|
||||
|
||||
srd3.u32All = srd;
|
||||
srd3.bits.INDEX_STRIDE = wave32 ? 2 : 3;
|
||||
|
||||
return srd3.u32All;
|
||||
}
|
||||
|
||||
return srd;
|
||||
}
|
||||
|
||||
extern "C" hsa_status_t (*fn_hsa_ven_amd_loader_query_host_address)(
|
||||
const void *device_address,
|
||||
const void **host_address);
|
||||
uint64_t ComputeQueue::GetKernelObjAddr(uint64_t addr) const {
|
||||
//TODO: convert dev_addr to host_addr
|
||||
uint64_t host_addr = 0;
|
||||
auto ret = fn_hsa_ven_amd_loader_query_host_address(reinterpret_cast<const void *>(addr),
|
||||
reinterpret_cast<const void **>(&host_addr));
|
||||
if (ret == HSA_STATUS_ERROR_INVALID_ARGUMENT) {
|
||||
return NULL;
|
||||
}
|
||||
|
||||
return host_addr;
|
||||
}
|
||||
|
||||
void ComputeQueue::RingDoorbell() {
|
||||
thread_cond_lock_.lock();
|
||||
thread_cond_lock_.unlock();
|
||||
debug_print("notify %p wptr=%" PRIx64 " rptr=%" PRIx64 "\n",
|
||||
ring, GetRingWptr()->load(), GetRingRptr()->load());
|
||||
thread_cond_.notify_one();
|
||||
}
|
||||
|
||||
hsa_status_t ComputeQueue::Init(void) {
|
||||
hsa_status_t ret = use_hws ? HwsInit() : SwsInit();
|
||||
if (ret)
|
||||
return ret;
|
||||
|
||||
ib_start_addr = cmdbuf_addr;
|
||||
cmdbuf_aql_frame_size = device->GetAqlFrameSize();
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
hsa_status_t ComputeQueue::Fini(void) {
|
||||
return use_hws ? HwsFini() : SwsFini();
|
||||
}
|
||||
|
||||
hsa_status_t ComputeQueue::PreSubmit(void) {
|
||||
if (!device->WaitPagingFence(this))
|
||||
return HSA_STATUS_ERROR;
|
||||
|
||||
RelocateCmdbufScratchBase(ib_start_addr);
|
||||
|
||||
return HSA_STATUS_SUCCESS;
|
||||
}
|
||||
|
||||
hsa_status_t ComputeQueue::EndSubmit(void) {
|
||||
// record last submitted cmdbuf_aql_frame_write_index to see if GPU is hungry
|
||||
sync_point = cmdbuf_aql_frame_write_index;
|
||||
|
||||
ib_start_addr = cmdbuf_addr +
|
||||
(cmdbuf_aql_frame_write_index % WDDMDevice::GetAqlFrameNum()) *
|
||||
cmdbuf_aql_frame_size;
|
||||
ib_size = 0;
|
||||
|
||||
return HSA_STATUS_SUCCESS;
|
||||
}
|
||||
|
||||
hsa_status_t ComputeQueue::Submit(void) {
|
||||
hsa_status_t ret = PreSubmit();
|
||||
if (ret)
|
||||
return HSA_STATUS_ERROR;
|
||||
|
||||
ret = use_hws ?
|
||||
HwsSubmit(ib_start_addr, ib_size, cmdbuf_aql_frame_write_index) :
|
||||
SwsSubmit(ib_start_addr, ib_size, cmdbuf_aql_frame_write_index);
|
||||
if (ret)
|
||||
return HSA_STATUS_ERROR;
|
||||
|
||||
ret = EndSubmit();
|
||||
if (ret)
|
||||
return HSA_STATUS_ERROR;
|
||||
|
||||
return HSA_STATUS_SUCCESS;
|
||||
}
|
||||
|
||||
hsa_status_t
|
||||
ComputeQueue::KernelDispatchAqlToPm4(char *cpu, hsa_kernel_dispatch_packet_t *packet) {
|
||||
debug_print("queue %p kernel dispatch head=%x setup=%x wx=%x wy=%x wz=%x "
|
||||
"gx=%x gy=%x gz=%x ps=%x gs=%x ko=%" PRIx64 " ka=%p cs=%" PRIx64 "\n",
|
||||
ring, packet->header,
|
||||
packet->setup, packet->workgroup_size_x, packet->workgroup_size_y,
|
||||
packet->workgroup_size_z, packet->grid_size_x, packet->grid_size_y,
|
||||
packet->grid_size_z, packet->private_segment_size,
|
||||
packet->group_segment_size, packet->kernel_object, packet->kernarg_address,
|
||||
packet->completion_signal.handle);
|
||||
|
||||
if (packet->workgroup_size_x > 1024 ||
|
||||
packet->workgroup_size_y > 1024 ||
|
||||
packet->workgroup_size_z > 1024)
|
||||
return HSA_STATUS_ERROR_INVALID_ARGUMENT;
|
||||
|
||||
int major = device->Major();
|
||||
int i = ib_size;
|
||||
|
||||
const amd_kernel_code_t* kernel_object =
|
||||
(const amd_kernel_code_t *)GetKernelObjAddr(packet->kernel_object);
|
||||
if (kernel_object == NULL) {
|
||||
return HSA_STATUS_ERROR_INVALID_CODE_OBJECT;
|
||||
}
|
||||
|
||||
void* entry = (void*)(packet->kernel_object + kernel_object->kernel_code_entry_byte_offset);
|
||||
assert((size_t)entry % AMD_ISA_ALIGN_BYTES == 0);
|
||||
|
||||
debug_print("kernel object property=%x entry=%p lds=%x+%x\n",
|
||||
kernel_object->kernel_code_properties, entry,
|
||||
kernel_object->workgroup_group_segment_byte_size,
|
||||
packet->group_segment_size);
|
||||
|
||||
if (packet->setup == 0 || packet->setup > 3)
|
||||
return HSA_STATUS_ERROR_INCOMPATIBLE_ARGUMENTS;
|
||||
if (packet->group_segment_size > device->LdsSize())
|
||||
return HSA_STATUS_ERROR_INVALID_ALLOCATION;
|
||||
|
||||
uint32_t lds_blks = device->LdsBlocks(packet);
|
||||
if (lds_blks > 128)
|
||||
return HSA_STATUS_ERROR_INVALID_ARGUMENT;
|
||||
|
||||
const bool wave32 =
|
||||
AMD_HSA_BITS_GET(kernel_object->kernel_code_properties,
|
||||
AMD_KERNEL_CODE_PROPERTIES_ENABLE_WAVEFRONT_SIZE32);
|
||||
|
||||
assert(packet->private_segment_size >= kernel_object->workitem_private_segment_byte_size);
|
||||
UpdateScratch(packet->private_segment_size, wave32);
|
||||
|
||||
amd_signal_t *signal = (amd_signal_t *)packet->completion_signal.handle;
|
||||
|
||||
// Record start timestamp when enabling profiling
|
||||
if (signal && EnableProfiling())
|
||||
i += cmd_util.BuildCopyData(&signal->start_ts, cpu + i);
|
||||
|
||||
// Build a barrier packet if it is requested
|
||||
const bool is_barrier_packet = (packet->header >> HSA_PACKET_HEADER_BARRIER) & 0x1;
|
||||
if (is_barrier_packet && needs_barrier)
|
||||
i += cmd_util.BuildBarrier(cpu + i);
|
||||
|
||||
// flush cache
|
||||
i += cmd_util.BuildAcquireMem(major, cpu + i);
|
||||
|
||||
if (major >= 11) {
|
||||
AppendCmdbufSratchBaseOffset(
|
||||
i + offsetof(struct SetScratchTemplate, scratch_lo));
|
||||
|
||||
i += cmd_util.BuildScratch(ScratchBase(), cpu + i);
|
||||
i += cmd_util.BuildComputeShaderParams(cpu + i);
|
||||
}
|
||||
|
||||
struct DispatchInfo info;
|
||||
info.major = major;
|
||||
info.pPacket = packet;
|
||||
info.pEntry = entry;
|
||||
info.pKernelObject = kernel_object;
|
||||
info.ldsBlks = lds_blks;
|
||||
info.pAmdQueue = amd_queue_;
|
||||
info.wave32 = wave32;
|
||||
info.srd = UpdateIndexStride(
|
||||
info.pAmdQueue->scratch_resource_descriptor[3], wave32);
|
||||
info.pScratchBase = ScratchBase();
|
||||
info.scratchSizePerWave = ScratchSizePerWave();
|
||||
memset(info.scratchBaseOffset, 0, sizeof(info.scratchBaseOffset));
|
||||
info.offsetCnt = 0;
|
||||
|
||||
size_t size;
|
||||
size = cmd_util.BuildDispatch(&info, cpu + i);
|
||||
for (int j = 0; j < info.offsetCnt; j++)
|
||||
AppendCmdbufSratchBaseOffset(i + info.scratchBaseOffset[j]);
|
||||
i += size;
|
||||
|
||||
needs_barrier = (packet->completion_signal.handle == 0);
|
||||
|
||||
if (signal) {
|
||||
// wait cs done
|
||||
i += cmd_util.BuildBarrier(cpu + i);
|
||||
|
||||
// Record end timestamp when enabling profiling
|
||||
if (EnableProfiling())
|
||||
i += cmd_util.BuildCopyData(&signal->end_ts, cpu + i);
|
||||
|
||||
// flush cache
|
||||
i += cmd_util.BuildAcquireMem(major, cpu + i);
|
||||
|
||||
assert(signal->kind == AMD_SIGNAL_KIND_USER);
|
||||
uint64_t *signal_addr = (uint64_t *)&signal->value;
|
||||
debug_print("signal value=%" PRIx64 "\n", signal->value);
|
||||
|
||||
i += cmd_util.BuildAtomicMem(signal_addr, TC_OP_ATOMIC_ADD_RTN_64, cpu + i, cache_policy__mec_atomic_mem__bypass, -1);
|
||||
}
|
||||
|
||||
// The ring_rptr is used to record pm4 queue rptr value,
|
||||
// dispatch readptr position, this is used to share rptr with
|
||||
// aql queue.
|
||||
i += cmd_util.BuildAtomicMem((uint64_t *)ring_rptr, TC_OP_ATOMIC_ADD_RTN_64, cpu + i);
|
||||
|
||||
ib_size = i;
|
||||
cmdbuf_aql_frame_write_index++;
|
||||
packet->header = HSA_PACKET_TYPE_INVALID;
|
||||
|
||||
return HSA_STATUS_SUCCESS;
|
||||
}
|
||||
|
||||
extern "C" hsa_signal_value_t (*fn_hsa_signal_load_relaxed)(
|
||||
hsa_signal_t signal);
|
||||
extern "C" hsa_signal_value_t (*fn_hsa_signal_wait_relaxed)(
|
||||
hsa_signal_t signal,
|
||||
hsa_signal_condition_t condition,
|
||||
hsa_signal_value_t compare_value,
|
||||
uint64_t timeout_hint,
|
||||
hsa_wait_state_t wait_state_hint);
|
||||
hsa_status_t
|
||||
ComputeQueue::BarrierGenericAqlToPm4(char *cpu, hsa_barrier_and_packet_t *packet, bool is_or) {
|
||||
debug_print("queue %p %s head=%x dep %" PRIx64 " %" PRIx64 " %" PRIx64
|
||||
" %" PRIx64 " %" PRIx64 " cs=%" PRIx64"\n",
|
||||
ring, is_or ? "or" : "and",
|
||||
packet->header, packet->dep_signal[0].handle,
|
||||
packet->dep_signal[1].handle, packet->dep_signal[2].handle,
|
||||
packet->dep_signal[3].handle, packet->dep_signal[4].handle,
|
||||
packet->completion_signal.handle);
|
||||
// fix me: can we use gpu packet?
|
||||
if (is_or) {
|
||||
bool unsignaled = true;
|
||||
hsa_signal_t sig[5];
|
||||
int n = 0;
|
||||
for (int i = 0; i < 5; i++) {
|
||||
if (packet->dep_signal[i].handle)
|
||||
sig[n++] = packet->dep_signal[i];
|
||||
}
|
||||
|
||||
while (n) {
|
||||
for (int i = 0; i < n; i++) {
|
||||
if (!fn_hsa_signal_load_relaxed(sig[i])) {
|
||||
unsignaled = false;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (!unsignaled)
|
||||
break;
|
||||
|
||||
std::this_thread::sleep_for(std::chrono::microseconds(20));
|
||||
}
|
||||
} else {
|
||||
for (int i = 0; i < 5; i++) {
|
||||
if (!packet->dep_signal[i].handle)
|
||||
continue;
|
||||
|
||||
hsa_signal_value_t value =
|
||||
fn_hsa_signal_wait_relaxed(packet->dep_signal[i], HSA_SIGNAL_CONDITION_EQ, 0, UINT64_MAX, HSA_WAIT_STATE_BLOCKED);
|
||||
assert(value == 0);
|
||||
}
|
||||
}
|
||||
|
||||
int major = device->Major();
|
||||
int i = ib_size;
|
||||
|
||||
if (packet->completion_signal.handle != 0) {
|
||||
amd_signal_t *signal = (amd_signal_t *)packet->completion_signal.handle;
|
||||
assert(signal->kind == AMD_SIGNAL_KIND_USER);
|
||||
uint64_t *signal_addr = (uint64_t *)&signal->value;
|
||||
debug_print("signal value=%" PRIx64 "\n", signal->value);
|
||||
|
||||
// Record start timestamp when enabling profiling
|
||||
if (EnableProfiling())
|
||||
i += cmd_util.BuildCopyData(&signal->start_ts, cpu + i);
|
||||
|
||||
if (needs_barrier)
|
||||
i += cmd_util.BuildBarrier(cpu + i);
|
||||
|
||||
needs_barrier = false;
|
||||
|
||||
// Record end timestamp when enabling profiling
|
||||
if (EnableProfiling())
|
||||
i += cmd_util.BuildCopyData(&signal->end_ts, cpu + i);
|
||||
|
||||
// flush cache
|
||||
i += cmd_util.BuildAcquireMem(major, cpu + i);
|
||||
|
||||
i += cmd_util.BuildAtomicMem(signal_addr, TC_OP_ATOMIC_ADD_RTN_64, cpu + i, cache_policy__mec_atomic_mem__bypass, -1);
|
||||
}
|
||||
|
||||
// The ring_rptr is used to record pm4 queue rptr value,
|
||||
// dispatch readptr position, this is used to share rptr with
|
||||
// aql queue.
|
||||
i += cmd_util.BuildAtomicMem((uint64_t *)ring_rptr, TC_OP_ATOMIC_ADD_RTN_64, cpu + i);
|
||||
|
||||
ib_size = i;
|
||||
cmdbuf_aql_frame_write_index++;
|
||||
packet->header = HSA_PACKET_TYPE_INVALID;
|
||||
|
||||
return HSA_STATUS_SUCCESS;
|
||||
}
|
||||
|
||||
hsa_status_t ComputeQueue::VendorSpecificAqlToPm4(char *cpu, amd_aql_pm4_ib *packet) {
|
||||
constexpr uint32_t AMD_AQL_FORMAT_PM4_IB = 0x1;
|
||||
assert(packet->ven_hdr == AMD_AQL_FORMAT_PM4_IB);
|
||||
|
||||
uint8_t op = (packet->ib_jump_cmd[0] >> PM4_OPCODE_SHIFT) & 0xff;
|
||||
assert(op == IT_INDIRECT_BUFFER);
|
||||
uint32_t* pm4_addr = reinterpret_cast<uint32_t*>((static_cast<uint64_t>(packet->ib_jump_cmd[2]) << 32) | (static_cast<uint64_t>(packet->ib_jump_cmd[1]) & ~3ull));
|
||||
uint32_t pm4_size = packet->ib_jump_cmd[3]&0xfffff;
|
||||
debug_print("queue %p %s VENDOR_SPECIFIC pkt pm4_addr %p pm4_size %" PRIx64 " cs=%" PRIx64"\n",
|
||||
ring, vendor_packet_support ? "process" : "skip", pm4_addr, pm4_size,
|
||||
packet->completion_signal.handle);
|
||||
for (int i = 0; i < pm4_size; i++) {
|
||||
debug_print("pm4_addr[%d]=%#x\n", i, pm4_addr[i]);
|
||||
}
|
||||
|
||||
if (vendor_packet_support) {
|
||||
int major = device->Major();
|
||||
int i = ib_size;
|
||||
|
||||
memcpy(cpu+i, pm4_addr, pm4_size * sizeof(uint32_t));
|
||||
i += pm4_size * sizeof(uint32_t);
|
||||
|
||||
if (packet->completion_signal.handle != 0) {
|
||||
amd_signal_t *signal = (amd_signal_t *)packet->completion_signal.handle;
|
||||
assert(signal->kind == AMD_SIGNAL_KIND_USER);
|
||||
uint64_t *signal_addr = (uint64_t *)&signal->value;
|
||||
debug_print("signal value=%" PRIx64 "\n", signal->value);
|
||||
|
||||
// Record start timestamp when enabling profiling
|
||||
if (EnableProfiling())
|
||||
i += cmd_util.BuildCopyData(&signal->start_ts, cpu + i);
|
||||
|
||||
//if (needs_barrier)
|
||||
i += cmd_util.BuildBarrier(cpu + i);
|
||||
|
||||
//needs_barrier = false;
|
||||
|
||||
// Record end timestamp when enabling profiling
|
||||
if (EnableProfiling())
|
||||
i += cmd_util.BuildCopyData(&signal->end_ts, cpu + i);
|
||||
|
||||
// flush cache
|
||||
i += cmd_util.BuildAcquireMem(major, cpu + i);
|
||||
|
||||
i += cmd_util.BuildAtomicMem(signal_addr, TC_OP_ATOMIC_ADD_RTN_64, cpu + i, cache_policy__mec_atomic_mem__bypass, -1);
|
||||
}
|
||||
|
||||
// The ring_rptr is used to record pm4 queue rptr value,
|
||||
// dispatch readptr position, this is used to share rptr with
|
||||
// aql queue.
|
||||
i += cmd_util.BuildAtomicMem((uint64_t *)ring_rptr, TC_OP_ATOMIC_ADD_RTN_64, cpu + i);
|
||||
|
||||
ib_size = i;
|
||||
} else {
|
||||
if (packet->completion_signal.handle != 0) {
|
||||
fn_hsa_signal_store_screlease(packet->completion_signal, 0);
|
||||
}
|
||||
}
|
||||
|
||||
cmdbuf_aql_frame_write_index++;
|
||||
packet->header = HSA_PACKET_TYPE_INVALID;
|
||||
return HSA_STATUS_SUCCESS;
|
||||
}
|
||||
|
||||
hsa_status_t ComputeQueue::SwitchAql2PM4(void) {
|
||||
|
||||
uint16_t *packet = (uint16_t *) ((char *)ring +
|
||||
(cmdbuf_aql_frame_write_index % ring_size) * 64);
|
||||
uint16_t header = (*packet >> HSA_PACKET_HEADER_TYPE);
|
||||
header &= (1 << HSA_PACKET_HEADER_WIDTH_TYPE) - 1;
|
||||
hsa_kernel_dispatch_packet_t *aql_packet =
|
||||
(hsa_kernel_dispatch_packet_t *)packet;
|
||||
hsa_status_t ret;
|
||||
|
||||
switch (header) {
|
||||
case HSA_PACKET_TYPE_KERNEL_DISPATCH:
|
||||
ret = KernelDispatchAqlToPm4((char *)ib_start_addr, aql_packet);
|
||||
if (ret != HSA_STATUS_SUCCESS)
|
||||
return ret;
|
||||
|
||||
// Stop merging packages util below conditions are met:
|
||||
// 1) The kernel with completion signal;
|
||||
// 2) The cmdbuf_aql_frame_write_index reaches the end of cmdbuf
|
||||
// 3) The queue is empty now, submit the package right now.
|
||||
if (!(aql_packet->completion_signal.handle) &&
|
||||
(cmdbuf_aql_frame_write_index % WDDMDevice::GetAqlFrameNum()) &&
|
||||
(*sync_addr != sync_point))
|
||||
return HSA_STATUS_SUCCESS;
|
||||
|
||||
break;
|
||||
case HSA_PACKET_TYPE_BARRIER_AND:
|
||||
BarrierGenericAqlToPm4((char *)ib_start_addr, (hsa_barrier_and_packet_t *)aql_packet);
|
||||
break;
|
||||
case HSA_PACKET_TYPE_BARRIER_OR:
|
||||
BarrierGenericAqlToPm4((char *)ib_start_addr, (hsa_barrier_and_packet_t *)aql_packet, true);
|
||||
break;
|
||||
case HSA_PACKET_TYPE_VENDOR_SPECIFIC:
|
||||
VendorSpecificAqlToPm4((char *)ib_start_addr, (amd_aql_pm4_ib *)aql_packet);
|
||||
break;
|
||||
case HSA_PACKET_TYPE_INVALID:
|
||||
// When packets are submitted out of order, the format field of current AQL packet
|
||||
// may not have been updated yet and is still INVALID. Return HSA_STATUS_SUCCESS and
|
||||
// do not process AQL packets before the packet format field is updated.
|
||||
assert(false && "Should not reach here, HSA_PACKET_TYPE_INVALID has been filtered in upper layer");
|
||||
return HSA_STATUS_SUCCESS;
|
||||
default:
|
||||
return HSA_STATUS_ERROR_INVALID_PACKET_FORMAT;
|
||||
}
|
||||
|
||||
ready_to_submit = true;
|
||||
|
||||
return HSA_STATUS_SUCCESS;
|
||||
}
|
||||
|
||||
hsa_status_t ComputeQueue::Process(void) {
|
||||
|
||||
while (cmdbuf_aql_frame_write_index < ring_wptr->load() &&
|
||||
!IsInvalidPacket()) {
|
||||
debug_print("process %p wptr=%" PRIx64 " rptr=%" PRIx64 "\n",
|
||||
ring, ring_wptr->load(), ring_rptr->load());
|
||||
|
||||
hsa_status_t ret;
|
||||
|
||||
// wait for next few cmdbuf slots to be free
|
||||
// If wptr catch up the rptr in the cmdbuf, this needs wait for the rptr to free the cmdbuf.
|
||||
// Here the wptr comes from queue->cmdbuf_aql_frame_write_index, while rptr comes from *queue->sync_addr.
|
||||
if (*sync_addr + WDDMDevice::GetAqlFrameNum() <= cmdbuf_aql_frame_write_index) {
|
||||
uint64_t value = cmdbuf_aql_frame_write_index - WDDMDevice::GetAqlFrameNum() + 1;
|
||||
if (!device->CpuWait(&syncobj, &value, 1, false))
|
||||
return HSA_STATUS_ERROR;
|
||||
}
|
||||
|
||||
ret = SwitchAql2PM4();
|
||||
if (ret != HSA_STATUS_SUCCESS)
|
||||
return ret;
|
||||
|
||||
if (!ready_to_submit)
|
||||
continue;
|
||||
|
||||
ret = Submit();
|
||||
if (ret != HSA_STATUS_SUCCESS)
|
||||
return ret;
|
||||
|
||||
ready_to_submit = false;
|
||||
|
||||
debug_print("done %p wptr=%" PRIx64 " rptr=%" PRIx64 "\n",
|
||||
ring, ring_wptr->load(), ring_rptr->load());
|
||||
|
||||
}
|
||||
|
||||
return HSA_STATUS_SUCCESS;
|
||||
}
|
||||
|
||||
hsa_status_t SDMAQueue::Init(void) {
|
||||
hsa_status_t ret = use_hws ? HwsInit() : SwsInit();
|
||||
if (ret)
|
||||
return ret;
|
||||
|
||||
std::memset((char *)cmdbuf_addr, 0, cmdbuf_size);
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
hsa_status_t SDMAQueue::Fini(void) {
|
||||
return use_hws ? HwsFini() : SwsFini();
|
||||
}
|
||||
|
||||
int SDMAQueue::PreparePacket(uint32_t offset, uint64_t size) {
|
||||
ib_start_addr = cmdbuf_addr + offset;
|
||||
ib_size = size;
|
||||
rptr_next += ib_size;
|
||||
|
||||
return STATUS_SUCCESS;
|
||||
}
|
||||
|
||||
hsa_status_t SDMAQueue::Submit(void) {
|
||||
if (!device->WaitPagingFence(this))
|
||||
return HSA_STATUS_ERROR;
|
||||
|
||||
int ret = use_hws ?
|
||||
HwsSubmit(ib_start_addr, ib_size, rptr_next) :
|
||||
SwsSubmit(ib_start_addr, ib_size, rptr_next);
|
||||
if (ret)
|
||||
return HSA_STATUS_ERROR;
|
||||
|
||||
return HSA_STATUS_SUCCESS;
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,163 @@
|
||||
#include <cassert>
|
||||
#include <map>
|
||||
#include <algorithm>
|
||||
#include "inc/wddm/va_mgr.h"
|
||||
|
||||
using namespace std;
|
||||
|
||||
namespace rocr {
|
||||
namespace core {
|
||||
|
||||
VaMgr::VaMgr(uint64_t start, uint64_t size, uint64_t min_align) {
|
||||
min_align_ = min_align;
|
||||
auto free_it = free_list_.insert(make_pair(size, start));
|
||||
frag_map_[start] = make_fragment(free_it, size);
|
||||
}
|
||||
|
||||
VaMgr::~VaMgr() {
|
||||
|
||||
assert(free_list_.size() == 1);
|
||||
assert(frag_map_.size() == 1);
|
||||
|
||||
free_list_.clear();
|
||||
frag_map_.clear();
|
||||
}
|
||||
|
||||
uint64_t VaMgr::Alloc(uint64_t bytes, uint64_t align, uint64_t addr) {
|
||||
|
||||
if (addr > 0 &&
|
||||
(align == 0 || (addr % align) == 0)) {
|
||||
|
||||
lock_guard<mutex> gard(lock_);
|
||||
auto frag_it = frag_map_.upper_bound(addr);
|
||||
assert(frag_it != frag_map_.begin());
|
||||
--frag_it;
|
||||
|
||||
while (frag_it != frag_map_.begin()) {
|
||||
const uint64_t base = frag_it->first;
|
||||
const uint64_t size = frag_it->second.size;
|
||||
|
||||
// Cannot find free fragment contains the target `addr`
|
||||
if (bytes > size || addr < base || addr + bytes >= base + size ||
|
||||
!is_free(frag_it->second)) {
|
||||
--frag_it;
|
||||
continue;
|
||||
} else if (addr >= base + size)
|
||||
break;
|
||||
|
||||
|
||||
// Try to allocate target `addr` from this free fragment
|
||||
auto free_it = frag_it->second.free_list_entry_;
|
||||
assert(free_it != free_list_.end());
|
||||
|
||||
free_list_.erase(free_it);
|
||||
frag_it->second.size = bytes;
|
||||
set_used(frag_it->second);
|
||||
|
||||
// [base, addr)
|
||||
if (addr > base) add_free_fragment(addr - base, base);
|
||||
|
||||
// [addr, addr + bytes) is used
|
||||
|
||||
// [addr + bytes, base + size)
|
||||
if (base + size > addr + bytes) add_free_fragment(base + size - addr - bytes, addr + bytes);
|
||||
|
||||
return addr;
|
||||
}
|
||||
}
|
||||
|
||||
// Allocate not fixed address
|
||||
return AllocImpl(bytes, align);
|
||||
}
|
||||
|
||||
uint64_t VaMgr::AllocImpl(const uint64_t bytes, const uint64_t align) {
|
||||
uint64_t addr = 0;
|
||||
uint64_t align_bytes = bytes;
|
||||
const int retry = align == 0 ? 0 : 1;
|
||||
const uint64_t new_align = align == 0 ? min_align_ : AlignUp(align, min_align_);
|
||||
|
||||
lock_guard<mutex> gard(lock_);
|
||||
for (int i = 0; i <= retry; i++) {
|
||||
auto free_it = free_list_.lower_bound(align_bytes);
|
||||
if (free_it == free_list_.end()) break;
|
||||
|
||||
uint64_t base = free_it->second;
|
||||
uint64_t size = free_it->first;
|
||||
|
||||
assert(size >= align_bytes);
|
||||
|
||||
auto fragment = frag_map_.find(base);
|
||||
|
||||
assert(fragment != frag_map_.end());
|
||||
assert(size == fragment->second.size);
|
||||
|
||||
uint64_t delta = align == 0 ? 0 : base % align;
|
||||
if (delta == 0) {
|
||||
// already find aligned address
|
||||
addr = base;
|
||||
|
||||
free_list_.erase(free_it);
|
||||
fragment->second.size = bytes;
|
||||
set_used(fragment->second);
|
||||
|
||||
if (size > bytes) add_free_fragment(size - bytes, base + bytes);
|
||||
|
||||
break;
|
||||
} else if (i == 0) {
|
||||
align_bytes += new_align;
|
||||
continue;
|
||||
} else {
|
||||
uint64_t aligned_base = base + align - delta;
|
||||
addr = aligned_base;
|
||||
|
||||
free_list_.erase(free_it);
|
||||
|
||||
add_used_fragment(bytes, aligned_base);
|
||||
add_free_fragment(aligned_base - base, base);
|
||||
|
||||
if (size > aligned_base - base + bytes)
|
||||
add_free_fragment(size - (aligned_base - base) - bytes, aligned_base + bytes);
|
||||
|
||||
break;
|
||||
}
|
||||
}
|
||||
return addr;
|
||||
}
|
||||
|
||||
void VaMgr::Free(uint64_t addr) {
|
||||
if (addr == 0) return;
|
||||
|
||||
lock_guard<mutex> gard(lock_);
|
||||
auto frag_it = frag_map_.find(addr);
|
||||
if (frag_it == frag_map_.end() || is_free(frag_it->second)) return;
|
||||
|
||||
uint64_t base = addr;
|
||||
// Merge lower
|
||||
if (frag_it != frag_map_.begin()) {
|
||||
auto lower = frag_it;
|
||||
--lower;
|
||||
if (is_free(lower->second)) {
|
||||
remove_free_list_entry(lower->second);
|
||||
base -= lower->second.size;
|
||||
lower->second.size += frag_it->second.size;
|
||||
frag_map_.erase(frag_it);
|
||||
frag_it = lower;
|
||||
}
|
||||
}
|
||||
// Merge upper
|
||||
{
|
||||
auto upper = frag_it;
|
||||
++upper;
|
||||
if (upper != frag_map_.end() && is_free(upper->second)) {
|
||||
remove_free_list_entry(upper->second);
|
||||
frag_it->second.size += upper->second.size;
|
||||
frag_map_.erase(upper);
|
||||
}
|
||||
}
|
||||
uint64_t size = frag_it->second.size;
|
||||
auto it = free_list_.insert(make_pair(size, base));
|
||||
set_free(frag_it->second, it);
|
||||
}
|
||||
|
||||
} // namespace core
|
||||
} // namespace rocr
|
||||
Atsaukties uz šo jaunā problēmā
Block a user