From db09beaa082b097147ce2d8386fa76140293e0fc Mon Sep 17 00:00:00 2001 From: shaoyunl Date: Tue, 16 Jul 2019 16:46:39 -0400 Subject: [PATCH] KFDTest: remove the usage global g_TestGPUFamilyId Adjust the KFDTest for multi-gou support Change-Id: Ib3491e3f645d35fdba6ab702d65fcc86f48d3958 Signed-off-by: shaoyunl [ROCm/ROCR-Runtime commit: b4e834ab616451690a75519e9bb3e3a271b09453] --- .../rocr-runtime/tests/kfdtest/src/BasePacket.cpp | 5 +++++ .../rocr-runtime/tests/kfdtest/src/BasePacket.hpp | 3 ++- .../rocr-runtime/tests/kfdtest/src/BaseQueue.cpp | 2 ++ .../rocr-runtime/tests/kfdtest/src/BaseQueue.hpp | 2 ++ .../rocr-runtime/tests/kfdtest/src/Dispatch.cpp | 15 ++++++++++----- .../rocr-runtime/tests/kfdtest/src/Dispatch.hpp | 1 + .../tests/kfdtest/src/KFDBaseComponentTest.cpp | 10 ++++++++-- .../tests/kfdtest/src/KFDBaseComponentTest.hpp | 3 +++ .../tests/kfdtest/src/KFDEventTest.cpp | 5 +++-- .../tests/kfdtest/src/KFDTestFlags.hpp | 1 - .../tests/kfdtest/src/KFDTestMain.cpp | 1 + .../tests/kfdtest/src/KFDTestUtilQueue.cpp | 4 ++-- .../rocr-runtime/tests/kfdtest/src/PM4Packet.cpp | 5 ++++- .../rocr-runtime/tests/kfdtest/src/PM4Queue.cpp | 4 ++-- .../rocr-runtime/tests/kfdtest/src/SDMAPacket.cpp | 2 +- .../rocr-runtime/tests/kfdtest/src/SDMAQueue.cpp | 2 +- 16 files changed, 47 insertions(+), 18 deletions(-) diff --git a/projects/rocr-runtime/tests/kfdtest/src/BasePacket.cpp b/projects/rocr-runtime/tests/kfdtest/src/BasePacket.cpp index fd714ca756..daae6691ae 100644 --- a/projects/rocr-runtime/tests/kfdtest/src/BasePacket.cpp +++ b/projects/rocr-runtime/tests/kfdtest/src/BasePacket.cpp @@ -23,6 +23,11 @@ #include "BasePacket.hpp" #include "KFDTestUtil.hpp" +#include "KFDBaseComponentTest.hpp" + +BasePacket::BasePacket(void) { + m_FamilyId = g_baseTest->GetFamilyIdFromDefaultNode(); +} void BasePacket::Dump() const { unsigned int size = SizeInDWords(); diff --git a/projects/rocr-runtime/tests/kfdtest/src/BasePacket.hpp b/projects/rocr-runtime/tests/kfdtest/src/BasePacket.hpp index 1802f901df..efbebc88f1 100644 --- a/projects/rocr-runtime/tests/kfdtest/src/BasePacket.hpp +++ b/projects/rocr-runtime/tests/kfdtest/src/BasePacket.hpp @@ -37,7 +37,7 @@ enum PACKETTYPE { // @class BasePacket class BasePacket { public: - BasePacket(void) {} + BasePacket(void); virtual ~BasePacket(void) {} // @returns Packet type @@ -52,6 +52,7 @@ class BasePacket { void Dump() const; protected: + unsigned int m_FamilyId; }; #endif // __KFD_BASE_PACKET__H__ diff --git a/projects/rocr-runtime/tests/kfdtest/src/BaseQueue.cpp b/projects/rocr-runtime/tests/kfdtest/src/BaseQueue.cpp index 4aa9e843d2..835ab4c71d 100644 --- a/projects/rocr-runtime/tests/kfdtest/src/BaseQueue.cpp +++ b/projects/rocr-runtime/tests/kfdtest/src/BaseQueue.cpp @@ -26,6 +26,7 @@ #include "PM4Queue.hpp" #include "AqlQueue.hpp" #include "hsakmt.h" +#include "KFDBaseComponentTest.hpp" BaseQueue::BaseQueue() :m_QueueBuf(NULL), @@ -81,6 +82,7 @@ HSAKMT_STATUS BaseQueue::Create(unsigned int NodeId, unsigned int size, HSAuint6 m_pendingWptr = 0; m_pendingWptr64 = 0; m_Node = NodeId; + m_FamilyId = g_baseTest->GetFamilyIdFromNodeId(NodeId); return status; } diff --git a/projects/rocr-runtime/tests/kfdtest/src/BaseQueue.hpp b/projects/rocr-runtime/tests/kfdtest/src/BaseQueue.hpp index 0e5b946290..570af95c8c 100644 --- a/projects/rocr-runtime/tests/kfdtest/src/BaseQueue.hpp +++ b/projects/rocr-runtime/tests/kfdtest/src/BaseQueue.hpp @@ -87,6 +87,7 @@ class BaseQueue { HSAuint64 GetPendingWptr64() { return m_pendingWptr64; } virtual _HSA_QUEUE_TYPE GetQueueType() = 0; unsigned int GetNodeId() { return m_Node; } + unsigned int GetFamilyId() { return m_FamilyId; } protected: static const unsigned int CMD_NOP_TYPE_2 = 0x80000000; @@ -98,6 +99,7 @@ class BaseQueue { HsaQueueResource m_Resources; HsaMemoryBuffer *m_QueueBuf; unsigned int m_Node; + unsigned int m_FamilyId; // @return Write pointer modulo queue size in dwords virtual unsigned int Wptr() = 0; diff --git a/projects/rocr-runtime/tests/kfdtest/src/Dispatch.cpp b/projects/rocr-runtime/tests/kfdtest/src/Dispatch.cpp index 802841060c..559594ee44 100644 --- a/projects/rocr-runtime/tests/kfdtest/src/Dispatch.cpp +++ b/projects/rocr-runtime/tests/kfdtest/src/Dispatch.cpp @@ -28,6 +28,8 @@ #include "asic_reg/gfx_7_2_d.h" #include "asic_reg/gfx_7_2_sh_mask.h" +#include "KFDBaseComponentTest.hpp" + Dispatch::Dispatch(const HsaMemoryBuffer& isaBuf, const bool eventAutoReset) :m_IsaBuf(isaBuf), m_IndirectBuf(PACKETTYPE_PM4, PAGE_SIZE / sizeof(unsigned int), isaBuf.Node()), m_DimX(1), m_DimY(1), m_DimZ(1), m_pArg1(NULL), m_pArg2(NULL), m_pEop(NULL), m_ScratchEn(false), @@ -39,6 +41,8 @@ Dispatch::Dispatch(const HsaMemoryBuffer& isaBuf, const bool eventAutoReset) eventDesc.SyncVar.SyncVarSize = 0; hsaKmtCreateEvent(&eventDesc, !eventAutoReset, false, &m_pEop); + + m_FamilyId = g_baseTest->GetFamilyIdFromNodeId(isaBuf.Node()); } Dispatch::~Dispatch() { @@ -69,6 +73,7 @@ void Dispatch::SetSpiPriority(unsigned int priority) { void Dispatch::Submit(BaseQueue& queue) { ASSERT_NE(m_pEop, (void*)0); + EXPECT_EQ(m_FamilyId, queue.GetFamilyId()); BuildIb(); @@ -80,7 +85,7 @@ void Dispatch::Submit(BaseQueue& queue) { EventData.EventData.SyncVar.SyncVar.UserData, m_pEop->EventId)); } - queue.PlaceAndSubmitPacket(PM4ReleaseMemoryPacket(g_TestGPUFamilyId, false, m_pEop->EventData.HWData2, m_pEop->EventId)); + queue.PlaceAndSubmitPacket(PM4ReleaseMemoryPacket(m_FamilyId, false, m_pEop->EventData.HWData2, m_pEop->EventId)); if (!queue.GetSkipWaitConsump()) queue.Wait4PacketConsumption(); @@ -200,14 +205,14 @@ void Dispatch::BuildIb() { // ORDERED_APPEND_MODE=0, USE_THREAD_DIMENSIONS=1, ORDER_MODE=0, DISPATCH_CACHE_CNTL=0, // SCALAR_L1_INV_VOL=0, VECTOR_L1_INV_VOL=0, DATA_ATC=?, RESTORE=0} - m_IndirectBuf.AddPacket(PM4AcquireMemoryPacket(g_TestGPUFamilyId)); + m_IndirectBuf.AddPacket(PM4AcquireMemoryPacket(m_FamilyId)); m_IndirectBuf.AddPacket(PM4SetShaderRegPacket(mmCOMPUTE_START_X, COMPUTE_DISPATCH_DIMS_VALUES, ARRAY_SIZE(COMPUTE_DISPATCH_DIMS_VALUES))); m_IndirectBuf.AddPacket(PM4SetShaderRegPacket(mmCOMPUTE_PGM_LO, - (g_TestGPUFamilyId >= FAMILY_AI) ? COMPUTE_PGM_VALUES_GFX9 : COMPUTE_PGM_VALUES_GFX8, - (g_TestGPUFamilyId >= FAMILY_AI) ? ARRAY_SIZE(COMPUTE_PGM_VALUES_GFX9) : ARRAY_SIZE(COMPUTE_PGM_VALUES_GFX8))); + (m_FamilyId >= FAMILY_AI) ? COMPUTE_PGM_VALUES_GFX9 : COMPUTE_PGM_VALUES_GFX8, + (m_FamilyId >= FAMILY_AI) ? ARRAY_SIZE(COMPUTE_PGM_VALUES_GFX9) : ARRAY_SIZE(COMPUTE_PGM_VALUES_GFX8))); m_IndirectBuf.AddPacket(PM4SetShaderRegPacket(mmCOMPUTE_PGM_RSRC1, COMPUTE_PGM_RSRC, ARRAY_SIZE(COMPUTE_PGM_RSRC))); @@ -225,5 +230,5 @@ void Dispatch::BuildIb() { m_IndirectBuf.AddPacket(PM4PartialFlushPacket()); - m_IndirectBuf.AddPacket(PM4AcquireMemoryPacket(g_TestGPUFamilyId)); + m_IndirectBuf.AddPacket(PM4AcquireMemoryPacket(m_FamilyId)); } diff --git a/projects/rocr-runtime/tests/kfdtest/src/Dispatch.hpp b/projects/rocr-runtime/tests/kfdtest/src/Dispatch.hpp index b89306c02d..ce8ffba21b 100644 --- a/projects/rocr-runtime/tests/kfdtest/src/Dispatch.hpp +++ b/projects/rocr-runtime/tests/kfdtest/src/Dispatch.hpp @@ -70,6 +70,7 @@ class Dispatch { HSAuint64 m_scratch_base; unsigned int m_SpiPriority; + unsigned int m_FamilyId; }; #endif // __KFD_DISPATCH__H__ diff --git a/projects/rocr-runtime/tests/kfdtest/src/KFDBaseComponentTest.cpp b/projects/rocr-runtime/tests/kfdtest/src/KFDBaseComponentTest.cpp index c56e0ea239..27f3db1cda 100644 --- a/projects/rocr-runtime/tests/kfdtest/src/KFDBaseComponentTest.cpp +++ b/projects/rocr-runtime/tests/kfdtest/src/KFDBaseComponentTest.cpp @@ -61,12 +61,12 @@ void KFDBaseComponentTest::SetUp() { const HsaNodeProperties *nodeProperties = m_NodeInfo.HsaDefaultGPUNodeProperties(); ASSERT_NOTNULL(nodeProperties) << "failed to get HSA default GPU Node properties"; - g_TestGPUFamilyId = FamilyIdFromNode(nodeProperties); + m_FamilyId = FamilyIdFromNode(nodeProperties); GetSdmaInfo(nodeProperties, &m_numSdmaEngines, &m_numSdmaXgmiEngines, &m_numSdmaQueuesPerEngine); - m_FamilyId = g_TestGPUFamilyId; + g_baseTest = this; ROUTINE_END } @@ -84,6 +84,7 @@ void KFDBaseComponentTest::TearDown() { EXPECT_SUCCESS(hsaKmtReleaseSystemProperties()); EXPECT_SUCCESS(hsaKmtCloseKFD()); + g_baseTest = NULL; ROUTINE_END } @@ -128,6 +129,11 @@ HSAuint64 KFDBaseComponentTest::GetVramSize(int defaultGPUNode) { return 0; } +unsigned int KFDBaseComponentTest::GetFamilyIdFromNodeId(unsigned int nodeId) +{ + return FamilyIdFromNode(m_NodeInfo.GetNodeProperties(nodeId)); +} + int KFDBaseComponentTest::FindDRMRenderNode(int gpuNode) { HsaNodeProperties *nodeProperties; _HSAKMT_STATUS status; diff --git a/projects/rocr-runtime/tests/kfdtest/src/KFDBaseComponentTest.hpp b/projects/rocr-runtime/tests/kfdtest/src/KFDBaseComponentTest.hpp index 867718e06d..fce5c0d884 100644 --- a/projects/rocr-runtime/tests/kfdtest/src/KFDBaseComponentTest.hpp +++ b/projects/rocr-runtime/tests/kfdtest/src/KFDBaseComponentTest.hpp @@ -55,6 +55,8 @@ class KFDBaseComponentTest : public testing::Test { // @brief Finds DRM Render node corresponding to gpuNode // @return DRM Render Node if successful or -1 on failure int FindDRMRenderNode(int gpuNode); + unsigned int GetFamilyIdFromNodeId(unsigned int nodeId); + unsigned int GetFamilyIdFromDefaultNode(){ return m_FamilyId; } protected: HsaVersionInfo m_VersionInfo; @@ -76,4 +78,5 @@ class KFDBaseComponentTest : public testing::Test { virtual void TearDown(); }; +extern KFDBaseComponentTest* g_baseTest; #endif // __KFD_BASE_COMPONENT_TEST__H__ diff --git a/projects/rocr-runtime/tests/kfdtest/src/KFDEventTest.cpp b/projects/rocr-runtime/tests/kfdtest/src/KFDEventTest.cpp index 9aebb53b46..ca16e871ff 100644 --- a/projects/rocr-runtime/tests/kfdtest/src/KFDEventTest.cpp +++ b/projects/rocr-runtime/tests/kfdtest/src/KFDEventTest.cpp @@ -147,8 +147,9 @@ class QueueAndSignalBenchmark { uint64_t startTime; PM4Queue queue; + unsigned int familyId = g_baseTest->GetFamilyIdFromNodeId(node); HsaEvent** pHsaEvent = reinterpret_cast(calloc(eventCount, sizeof(HsaEvent*))); - size_t packetSize = PM4ReleaseMemoryPacket(g_TestGPUFamilyId, false, 0, 0).SizeInBytes(); + size_t packetSize = PM4ReleaseMemoryPacket(familyId, false, 0, 0).SizeInBytes(); int qSize = fmax(PAGE_SIZE, pow2_round_up(packetSize*eventCount + 1)); time = 0; @@ -162,7 +163,7 @@ class QueueAndSignalBenchmark { if (r != HSAKMT_STATUS_SUCCESS) goto exit; - queue.PlacePacket(PM4ReleaseMemoryPacket(g_TestGPUFamilyId, false, pHsaEvent[i]->EventData.HWData2, pHsaEvent[i]->EventId)); + queue.PlacePacket(PM4ReleaseMemoryPacket(familyId, false, pHsaEvent[i]->EventData.HWData2, pHsaEvent[i]->EventId)); } startTime = gettime(); diff --git a/projects/rocr-runtime/tests/kfdtest/src/KFDTestFlags.hpp b/projects/rocr-runtime/tests/kfdtest/src/KFDTestFlags.hpp index ecd309b9c7..1514695890 100644 --- a/projects/rocr-runtime/tests/kfdtest/src/KFDTestFlags.hpp +++ b/projects/rocr-runtime/tests/kfdtest/src/KFDTestFlags.hpp @@ -30,7 +30,6 @@ extern unsigned int g_TestTimeOut; extern int g_TestNodeId; extern int g_TestDstNodeId; extern bool g_IsChildProcess; -extern unsigned int g_TestGPUFamilyId; // Each test should call TEST_START with the test custom profile and HW scheduling enum TESTPROFILE{ diff --git a/projects/rocr-runtime/tests/kfdtest/src/KFDTestMain.cpp b/projects/rocr-runtime/tests/kfdtest/src/KFDTestMain.cpp index 7715bd06a1..a8837f95c4 100644 --- a/projects/rocr-runtime/tests/kfdtest/src/KFDTestMain.cpp +++ b/projects/rocr-runtime/tests/kfdtest/src/KFDTestMain.cpp @@ -54,6 +54,7 @@ int g_TestNodeId; int g_TestDstNodeId; bool g_IsChildProcess; unsigned int g_TestGPUFamilyId; +class KFDBaseComponentTest *g_baseTest; GTEST_API_ int main(int argc, char **argv) { // Default values for run parameters diff --git a/projects/rocr-runtime/tests/kfdtest/src/KFDTestUtilQueue.cpp b/projects/rocr-runtime/tests/kfdtest/src/KFDTestUtilQueue.cpp index 991485fbd4..e1fe07d4df 100644 --- a/projects/rocr-runtime/tests/kfdtest/src/KFDTestUtilQueue.cpp +++ b/projects/rocr-runtime/tests/kfdtest/src/KFDTestUtilQueue.cpp @@ -103,7 +103,7 @@ class AsyncMPSQ { if (m_queueType == HSA_QUEUE_SDMA) return SDMATimePacket(0).SizeInBytes(); else if (m_queueType == HSA_QUEUE_COMPUTE) - return PM4ReleaseMemoryPacket(g_TestGPUFamilyId, 0, 0, 0, 0, 0).SizeInBytes(); + return PM4ReleaseMemoryPacket(m_queue->GetFamilyId(), 0, 0, 0, 0, 0).SizeInBytes(); return 0; } @@ -126,7 +126,7 @@ class AsyncMPSQ { PlacePacket(SDMATimePacket(addr)); else if (m_queueType == HSA_QUEUE_COMPUTE) PlacePacket( - PM4ReleaseMemoryPacket(g_TestGPUFamilyId, true, (HSAuint64)addr, 0, true, true)); + PM4ReleaseMemoryPacket(m_queue->GetFamilyId(), true, (HSAuint64)addr, 0, true, true)); else WARN() << "Unsupported queue type!" << std::endl; } diff --git a/projects/rocr-runtime/tests/kfdtest/src/PM4Packet.cpp b/projects/rocr-runtime/tests/kfdtest/src/PM4Packet.cpp index 503e1b799c..195256fbaf 100644 --- a/projects/rocr-runtime/tests/kfdtest/src/PM4Packet.cpp +++ b/projects/rocr-runtime/tests/kfdtest/src/PM4Packet.cpp @@ -26,6 +26,7 @@ #include #include "PM4Packet.hpp" #include "hsakmttypes.h" +#include "KFDBaseComponentTest.hpp" #include "asic_reg/gfx_7_2_enum.h" @@ -75,7 +76,7 @@ void PM4WriteDataPacket::InitPacket(unsigned int *destBuf, void *data) { PM4ReleaseMemoryPacket::PM4ReleaseMemoryPacket(unsigned int familyId, bool isPolling, uint64_t address, uint64_t data, bool is64bit, bool isTimeStamp):m_pPacketData(NULL) { - + m_FamilyId = familyId; if (familyId < FAMILY_AI) InitPacketCI(isPolling, address, data, is64bit, isTimeStamp); else if (familyId < FAMILY_NV) @@ -264,6 +265,7 @@ void PM4IndirectBufPacket::InitPacket(IndirectBuffer *pIb) { } PM4AcquireMemoryPacket::PM4AcquireMemoryPacket(unsigned int familyId):m_pPacketData(NULL) { + m_FamilyId = familyId; if (familyId < FAMILY_NV) InitPacketAI(); @@ -278,6 +280,7 @@ void PM4AcquireMemoryPacket::InitPacketAI(void) { pkt = reinterpret_cast(calloc(1, m_packetSize)); m_pPacketData = pkt; EXPECT_NOTNULL(m_pPacketData); + InitPM4Header(pkt->header, IT_ACQUIRE_MEM); pkt->bitfields2.coher_cntl = 0x28c00000; // copied from the way the HSART does this. pkt->bitfields2.engine = engine_acquire_mem_PFP_0; diff --git a/projects/rocr-runtime/tests/kfdtest/src/PM4Queue.cpp b/projects/rocr-runtime/tests/kfdtest/src/PM4Queue.cpp index 6613ec8909..9bc6402fac 100644 --- a/projects/rocr-runtime/tests/kfdtest/src/PM4Queue.cpp +++ b/projects/rocr-runtime/tests/kfdtest/src/PM4Queue.cpp @@ -56,7 +56,7 @@ unsigned int PM4Queue::RptrWhenConsumed() { void PM4Queue::SubmitPacket() { // m_pending Wptr is in dwords - if (g_TestGPUFamilyId < FAMILY_AI) { + if (m_FamilyId < FAMILY_AI) { // Pre-Vega10 uses 32-bit wptr and doorbell MemoryBarrier(); *m_Resources.Queue_write_ptr = m_pendingWptr; @@ -73,7 +73,7 @@ void PM4Queue::SubmitPacket() { void PM4Queue::Wait4PacketConsumption(HsaEvent *event, unsigned int timeOut) { if (event) { - PlaceAndSubmitPacket(PM4ReleaseMemoryPacket(g_TestGPUFamilyId, 0, + PlaceAndSubmitPacket(PM4ReleaseMemoryPacket(m_FamilyId, 0, event->EventData.HWData2, event->EventId, true)); diff --git a/projects/rocr-runtime/tests/kfdtest/src/SDMAPacket.cpp b/projects/rocr-runtime/tests/kfdtest/src/SDMAPacket.cpp index 81ed3ba73d..f78745037b 100644 --- a/projects/rocr-runtime/tests/kfdtest/src/SDMAPacket.cpp +++ b/projects/rocr-runtime/tests/kfdtest/src/SDMAPacket.cpp @@ -30,7 +30,7 @@ /* Byte/dword count in many SDMA packets is 1-based in AI, meaning a * count of 1 is encoded as 0. */ -#define SDMA_COUNT(c) (g_TestGPUFamilyId < FAMILY_AI ? (c) : (c)-1) +#define SDMA_COUNT(c) (m_FamilyId < FAMILY_AI ? (c) : (c)-1) SDMAWriteDataPacket::SDMAWriteDataPacket(void): packetData(NULL) { diff --git a/projects/rocr-runtime/tests/kfdtest/src/SDMAQueue.cpp b/projects/rocr-runtime/tests/kfdtest/src/SDMAQueue.cpp index a6b41ff31a..9b77f78310 100644 --- a/projects/rocr-runtime/tests/kfdtest/src/SDMAQueue.cpp +++ b/projects/rocr-runtime/tests/kfdtest/src/SDMAQueue.cpp @@ -63,7 +63,7 @@ unsigned int SDMAQueue::RptrWhenConsumed() { void SDMAQueue::SubmitPacket() { // m_pending Wptr is in dwords - if (g_TestGPUFamilyId < FAMILY_AI) { + if (m_FamilyId < FAMILY_AI) { // Pre-Vega10 uses 32-bit wptr and doorbell unsigned int wPtrInBytes = m_pendingWptr * sizeof(unsigned int); MemoryBarrier();