AI/rocm-systems
2
0
Форкнуть 0
Код Задачи Запросы на слияние Действия Пакеты Проекты Релизы Вики Активность
Files
09a01ee11ca8905935ca76772b7bbbaa5863ad90
rocm-systems/projects/rocr-runtime/libhsakmt/tests/kfdtest/src/KFDQMTest.cpp
T
Alysa Liu 5be4fddf06 kfdtest: Support blit kernel copy (#677) 
Add support for blit kernel copy.
Add GpuMemCopyTest test for KFDQMTest.
2026-01-07 16:48:11 -05:00

2915 строки
101 KiB
C++
Исходник Ответственный История

/*
* Copyright (C) 2014-2018 Advanced Micro Devices, Inc. All Rights Reserved.
*
* Permission is hereby granted, free of charge, to any person obtaining a
* copy of this software and associated documentation files (the "Software"),
* to deal in the Software without restriction, including without limitation
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
* and/or sell copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in
* all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
* OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
* ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
* OTHER DEALINGS IN THE SOFTWARE.
*
*/
#include <sys/time.h>
#include <sys/mman.h>
#include <vector>
#include <utility>
#include <mutex>
#include "KFDQMTest.hpp"
#include "PM4Queue.hpp"
#include "PM4Packet.hpp"
#include "SDMAPacket.hpp"
#include "XgmiOptimizedSDMAQueue.hpp"
#include "AqlQueue.hpp"
#include <algorithm>
#include "Dispatch.hpp"
const unsigned int FILL_VALUE = 0x01010101;
const unsigned int INIT_VALUE = 0x5A5A5A5A;
extern unsigned int g_TestGPUsNum;
void KFDQMTest::SetUp() {
ROUTINE_START
KFDBaseComponentTest::SetUp();
ROUTINE_END
}
void KFDQMTest::TearDown() {
ROUTINE_START
KFDBaseComponentTest::TearDown();
ROUTINE_END
}
void KFDQMTest::CreateDestroyCpQueue(int gpuNode) {
PM4Queue queue;
ASSERT_SUCCESS_GPU(queue.Create(gpuNode), gpuNode);
EXPECT_SUCCESS_GPU(queue.Destroy(), gpuNode);
}
TEST_F(KFDQMTest, CreateDestroyCpQueue) {
TEST_START(TESTPROFILE_RUNALL)
ASSERT_SUCCESS(KFDTestLaunch([this](int gpuNode) {
this->CreateDestroyCpQueue(gpuNode);
}));
TEST_END
}
void KFDQMTest::SubmitNopCpQueue(int gpuNode) {
PM4Queue queue;
HsaEvent *event;
ASSERT_SUCCESS_GPU(CreateQueueTypeEvent(false, false, gpuNode, &event), gpuNode);
ASSERT_SUCCESS_GPU(queue.Create(gpuNode), gpuNode);
queue.PlaceAndSubmitPacket(PM4NopPacket());
queue.Wait4PacketConsumption(event);
hsaKmtDestroyEvent(event);
EXPECT_SUCCESS_GPU(queue.Destroy(), gpuNode);
}
TEST_F(KFDQMTest, SubmitNopCpQueue) {
TEST_START(TESTPROFILE_RUNALL)
ASSERT_SUCCESS(KFDTestLaunch([this](int gpuNode) {
this->SubmitNopCpQueue(gpuNode);
}));
TEST_END
}
void KFDQMTest::SubmitPacketCpQueue(int gpuNode) {
HsaMemoryBuffer destBuf(PAGE_SIZE, gpuNode, false);
destBuf.Fill(0xFF);
HsaEvent *event;
ASSERT_SUCCESS_GPU(CreateQueueTypeEvent(false, false, gpuNode, &event), gpuNode);
PM4Queue queue;
ASSERT_SUCCESS_GPU(queue.Create(gpuNode), gpuNode);
queue.PlaceAndSubmitPacket(PM4WriteDataPacket(destBuf.As<unsigned int*>(), 0, 0));
queue.Wait4PacketConsumption(event);
EXPECT_TRUE_GPU(WaitOnValue(destBuf.As<unsigned int*>(), 0), gpuNode);
hsaKmtDestroyEvent(event);
EXPECT_SUCCESS_GPU(queue.Destroy(), gpuNode);
}
TEST_F(KFDQMTest, SubmitPacketCpQueue) {
TEST_START(TESTPROFILE_RUNALL)
ASSERT_SUCCESS(KFDTestLaunch([this](int gpuNode) {
this->SubmitPacketCpQueue(gpuNode);
}));
TEST_END
}
void KFDQMTest::AllCpQueues(int gpuNode) {
int gpuIndex = Get_NodeInfo()->HsaGPUindexFromGpuNode(gpuNode);
HSAuint32 m_FamilyId = GetFamilyIdFromNodeId(gpuNode);
HsaMemoryBuffer destBuf(PAGE_SIZE, gpuNode, false);
destBuf.Fill(0xFF);
unsigned int m_numCpQueues = Get_NumCpQueues(gpuIndex);
std::vector<PM4Queue> queues(m_numCpQueues);
for (unsigned int qidx = 0; qidx < m_numCpQueues; ++qidx)
ASSERT_SUCCESS_GPU(queues[qidx].Create(gpuNode), gpuNode) << " QueueId=" << qidx;
for (unsigned int qidx = 0; qidx < m_numCpQueues; ++qidx) {
queues[qidx].PlaceAndSubmitPacket(PM4WriteDataPacket(destBuf.As<unsigned int*>()+qidx*2, qidx, qidx));
queues[qidx].PlaceAndSubmitPacket(PM4ReleaseMemoryPacket(m_FamilyId, true, 0, 0));
queues[qidx].Wait4PacketConsumption();
EXPECT_TRUE_GPU(WaitOnValue(destBuf.As<unsigned int*>()+qidx*2, qidx), gpuNode);
}
for (unsigned int qidx = 0; qidx < m_numCpQueues; ++qidx)
EXPECT_SUCCESS_GPU(queues[qidx].Destroy(), gpuNode);
}
TEST_F(KFDQMTest, AllCpQueues) {
TEST_START(TESTPROFILE_RUNALL)
ASSERT_SUCCESS(KFDTestLaunch([this](int gpuNode) {
this->AllCpQueues(gpuNode);
}));
TEST_END
}
void KFDQMTest::CreateDestroySdmaQueue(int gpuNode) {
SDMAQueue queue;
ASSERT_SUCCESS_GPU(queue.Create(gpuNode), gpuNode);
EXPECT_SUCCESS_GPU(queue.Destroy(), gpuNode);
}
TEST_F(KFDQMTest, CreateDestroySdmaQueue) {
TEST_START(TESTPROFILE_RUNALL)
ASSERT_SUCCESS(KFDTestLaunch([this](int gpuNode) {
this->CreateDestroySdmaQueue(gpuNode);
}));
TEST_END
}
void KFDQMTest::SubmitNopSdmaQueue(int gpuNode) {
SDMAQueue queue;
ASSERT_SUCCESS_GPU(queue.Create(gpuNode), gpuNode);
queue.PlaceAndSubmitPacket(SDMANopPacket());
queue.Wait4PacketConsumption();
EXPECT_SUCCESS_GPU(queue.Destroy(), gpuNode);
}
TEST_F(KFDQMTest, SubmitNopSdmaQueue) {
TEST_START(TESTPROFILE_RUNALL)
ASSERT_SUCCESS(KFDTestLaunch([this](int gpuNode) {
this->SubmitNopSdmaQueue(gpuNode);
}));
TEST_END
}
void KFDQMTest::SubmitPacketSdmaQueue(int gpuNode) {
HsaMemoryBuffer destBuf(PAGE_SIZE, gpuNode, false);
destBuf.Fill(0xFF);
SDMAQueue queue;
ASSERT_SUCCESS_GPU(queue.Create(gpuNode), gpuNode);
queue.PlaceAndSubmitPacket(SDMAWriteDataPacket(queue.GetFamilyId(), destBuf.As<void *>(), 0x02020202));
queue.Wait4PacketConsumption();
EXPECT_TRUE_GPU(WaitOnValue(destBuf.As<unsigned int*>(), 0x02020202), gpuNode);
EXPECT_SUCCESS_GPU(queue.Destroy(), gpuNode);
}
TEST_F(KFDQMTest, SubmitPacketSdmaQueue) {
TEST_START(TESTPROFILE_RUNALL)
ASSERT_SUCCESS(KFDTestLaunch([this](int gpuNode) {
this->SubmitPacketSdmaQueue(gpuNode);
}));
TEST_END
}
void KFDQMTest::AllSdmaQueues(int gpuNode) {
int gpuIndex = Get_NodeInfo()->HsaGPUindexFromGpuNode(gpuNode);
unsigned int m_numSdmaEngines = Get_NumSdmaEngines(gpuIndex);
unsigned int m_numSdmaQueuesPerEngine = Get_NumSdmaSdmaQueuesPerEngine(gpuIndex);
int bufSize = PAGE_SIZE;
const unsigned int numSdmaQueues = m_numSdmaEngines * m_numSdmaQueuesPerEngine;
LOG() << "Regular SDMA engines number: " << m_numSdmaEngines
<< " SDMA queues per engine: " << m_numSdmaQueuesPerEngine << std::endl;
HsaMemoryBuffer destBuf(bufSize << 1 , gpuNode, false);
HsaMemoryBuffer srcBuf(bufSize, gpuNode, false);
destBuf.Fill(0xFF);
std::vector<SDMAQueue> queues(numSdmaQueues);
for (unsigned int qidx = 0; qidx < numSdmaQueues; ++qidx)
ASSERT_SUCCESS_GPU(queues[qidx].Create(gpuNode), gpuNode);
for (unsigned int qidx = 0; qidx < numSdmaQueues; ++qidx) {
destBuf.Fill(0x0);
srcBuf.Fill(qidx + 0xa0);
queues[qidx].PlaceAndSubmitPacket(
SDMACopyDataPacket(queues[qidx].GetFamilyId(), destBuf.As<unsigned int*>(), srcBuf.As<unsigned int*>(), bufSize));
queues[qidx].PlaceAndSubmitPacket(
SDMAWriteDataPacket(queues[qidx].GetFamilyId(), destBuf.As<unsigned int*>() + bufSize/4, 0x02020202));
queues[qidx].Wait4PacketConsumption();
EXPECT_TRUE_GPU(WaitOnValue(destBuf.As<unsigned int*>() + bufSize/4, 0x02020202), gpuNode);
EXPECT_SUCCESS_GPU(memcmp(
destBuf.As<unsigned int*>(), srcBuf.As<unsigned int*>(), bufSize), gpuNode);
}
for (unsigned int qidx = 0; qidx < numSdmaQueues; ++qidx)
EXPECT_SUCCESS_GPU(queues[qidx].Destroy(), gpuNode);
}
TEST_F(KFDQMTest, AllSdmaQueues) {
TEST_START(TESTPROFILE_RUNALL)
ASSERT_SUCCESS(KFDTestLaunch([this](int gpuNode) {
this->AllSdmaQueues(gpuNode);
}));
TEST_END
}
void KFDQMTest::AllXgmiSdmaQueues(int gpuNode) {
int gpuIndex = Get_NodeInfo()->HsaGPUindexFromGpuNode(gpuNode);
unsigned int m_numSdmaXgmiEngines = Get_NumSdmaSdmaXgmiEngines(gpuIndex);
unsigned int m_numSdmaQueuesPerEngine = Get_NumSdmaSdmaQueuesPerEngine(gpuIndex);
int bufSize = PAGE_SIZE;
int j;
const unsigned int numXgmiSdmaQueues =
m_numSdmaXgmiEngines * m_numSdmaQueuesPerEngine;
LOG() << "XGMI SDMA engines number: " << m_numSdmaXgmiEngines
<< " SDMA queues per engine: " << m_numSdmaQueuesPerEngine << std::endl;
HsaMemoryBuffer destBuf(bufSize << 1 , gpuNode, false);
HsaMemoryBuffer srcBuf(bufSize, gpuNode, false);
destBuf.Fill(0xFF);
std::vector<XgmiOptimizedSDMAQueue> xgmiSdmaQueues(numXgmiSdmaQueues);
for (j = 0; j < numXgmiSdmaQueues; ++j)
ASSERT_SUCCESS_GPU(xgmiSdmaQueues[j].Create(gpuNode), gpuNode);
for (j = 0; j < numXgmiSdmaQueues; ++j) {
destBuf.Fill(0x0);
srcBuf.Fill(j + 0xa0);
xgmiSdmaQueues[j].PlaceAndSubmitPacket(
SDMACopyDataPacket(xgmiSdmaQueues[j].GetFamilyId(),
destBuf.As<unsigned int*>(), srcBuf.As<unsigned int*>(), bufSize));
xgmiSdmaQueues[j].PlaceAndSubmitPacket(
SDMAWriteDataPacket(xgmiSdmaQueues[j].GetFamilyId(),
destBuf.As<unsigned int*>() + bufSize/4, 0x02020202));
xgmiSdmaQueues[j].Wait4PacketConsumption();
EXPECT_TRUE_GPU(WaitOnValue(destBuf.As<unsigned int*>() + bufSize/4, 0x02020202), gpuNode);
EXPECT_SUCCESS_GPU(memcmp(
destBuf.As<unsigned int*>(), srcBuf.As<unsigned int*>(), bufSize), gpuNode);
}
for (j = 0; j < numXgmiSdmaQueues; ++j)
EXPECT_SUCCESS_GPU(xgmiSdmaQueues[j].Destroy(), gpuNode);
}
TEST_F(KFDQMTest, AllXgmiSdmaQueues) {
TEST_START(TESTPROFILE_RUNALL)
ASSERT_SUCCESS(KFDTestLaunch([this](int gpuNode) {
this->AllXgmiSdmaQueues(gpuNode);
}));
TEST_END
}
void KFDQMTest::AllQueues(int gpuNode) {
int gpuIndex = Get_NodeInfo()->HsaGPUindexFromGpuNode(gpuNode);
HSAuint32 m_FamilyId = GetFamilyIdFromNodeId(gpuNode);
unsigned int m_numSdmaXgmiEngines = Get_NumSdmaSdmaXgmiEngines(gpuIndex);
unsigned int m_numSdmaQueuesPerEngine = Get_NumSdmaSdmaQueuesPerEngine(gpuIndex);
unsigned int m_numSdmaEngines = Get_NumSdmaEngines(gpuIndex);
unsigned int m_numCpQueues = Get_NumCpQueues(gpuIndex);
int bufSize = PAGE_SIZE;
unsigned int i, j;
const unsigned int numCpQueues = m_numCpQueues;
const unsigned int numSdmaQueues = m_numSdmaEngines * m_numSdmaQueuesPerEngine;
const unsigned int numXgmiSdmaQueues =
m_numSdmaXgmiEngines * m_numSdmaQueuesPerEngine;
HsaMemoryBuffer destBufCp(PAGE_SIZE, gpuNode, false);
destBufCp.Fill(0xFF);
HsaMemoryBuffer destBuf(bufSize << 1 , gpuNode, false);
HsaMemoryBuffer srcBuf(bufSize, gpuNode, false);
destBuf.Fill(0xFF);
std::vector<PM4Queue> cpQueues(numCpQueues);
std::vector<SDMAQueue> sdmaQueues(numSdmaQueues);
std::vector<XgmiOptimizedSDMAQueue> xgmiSdmaQueues(numXgmiSdmaQueues);
for (i = 0; i < numCpQueues; ++i)
ASSERT_SUCCESS_GPU(cpQueues[i].Create(gpuNode), gpuNode) << " QueueId=" << i;
for (j = 0; j < numSdmaQueues; ++j)
ASSERT_SUCCESS_GPU(sdmaQueues[j].Create(gpuNode), gpuNode);
for (j = 0; j < numXgmiSdmaQueues; ++j)
ASSERT_SUCCESS_GPU(xgmiSdmaQueues[j].Create(gpuNode), gpuNode);
for (i = 0; i < numCpQueues; ++i) {
cpQueues[i].PlaceAndSubmitPacket(PM4WriteDataPacket(destBufCp.As<unsigned int*>()+i*2, i, i));
cpQueues[i].PlaceAndSubmitPacket(PM4ReleaseMemoryPacket(m_FamilyId, true, 0, 0));
cpQueues[i].Wait4PacketConsumption();
EXPECT_TRUE_GPU(WaitOnValue(destBufCp.As<unsigned int*>()+i*2, i), gpuNode);
}
for (j = 0; j < numSdmaQueues; ++j) {
destBuf.Fill(0x0);
srcBuf.Fill(j + 0xa0);
sdmaQueues[j].PlaceAndSubmitPacket(
SDMACopyDataPacket(sdmaQueues[j].GetFamilyId(), destBuf.As<unsigned int*>(), srcBuf.As<unsigned int*>(), bufSize));
sdmaQueues[j].PlaceAndSubmitPacket(
SDMAWriteDataPacket(sdmaQueues[j].GetFamilyId(), destBuf.As<unsigned int*>() + bufSize/4, 0x02020202));
sdmaQueues[j].Wait4PacketConsumption();
EXPECT_TRUE_GPU(WaitOnValue(destBuf.As<unsigned int*>() + bufSize/4, 0x02020202), gpuNode);
EXPECT_SUCCESS_GPU(memcmp(
destBuf.As<unsigned int*>(), srcBuf.As<unsigned int*>(), bufSize), gpuNode);
}
for (j = 0; j < numXgmiSdmaQueues; ++j) {
destBuf.Fill(0x0);
srcBuf.Fill(j + 0xa0);
xgmiSdmaQueues[j].PlaceAndSubmitPacket(
SDMACopyDataPacket(xgmiSdmaQueues[j].GetFamilyId(),
destBuf.As<unsigned int*>(), srcBuf.As<unsigned int*>(), bufSize));
xgmiSdmaQueues[j].PlaceAndSubmitPacket(
SDMAWriteDataPacket(xgmiSdmaQueues[j].GetFamilyId(),
destBuf.As<unsigned int*>() + bufSize/4, 0x02020202));
xgmiSdmaQueues[j].Wait4PacketConsumption();
EXPECT_TRUE_GPU(WaitOnValue(destBuf.As<unsigned int*>() + bufSize/4, 0x02020202), gpuNode);
EXPECT_SUCCESS_GPU(memcmp(
destBuf.As<unsigned int*>(), srcBuf.As<unsigned int*>(), bufSize), gpuNode);
}
for (i = 0; i < numCpQueues; ++i)
EXPECT_SUCCESS_GPU(cpQueues[i].Destroy(), gpuNode);
for (j = 0; j < numSdmaQueues; ++j)
EXPECT_SUCCESS_GPU(sdmaQueues[j].Destroy(), gpuNode);
for (j = 0; j < numXgmiSdmaQueues; ++j)
EXPECT_SUCCESS_GPU(xgmiSdmaQueues[j].Destroy(), gpuNode);
}
TEST_F(KFDQMTest, AllQueues) {
TEST_START(TESTPROFILE_RUNALL)
ASSERT_SUCCESS(KFDTestLaunch([this](int gpuNode) {
this->AllQueues(gpuNode);
}));
TEST_END
}
/* The following test is designed to reproduce an intermittent hang on
* Fiji and other VI/Polaris GPUs. This test typically hangs in a few
* seconds. According to analysis done by HW engineers, the culprit
* seems to be PCIe speed switching. The problem can be worked around
* by disabling the lowest DPM level on Fiji.
*/
void KFDQMTest::SdmaConcurrentCopies(int gpuNode) {
int gpuIndex = Get_NodeInfo()->HsaGPUindexFromGpuNode(gpuNode);
HSAuint32 m_FamilyId = GetFamilyIdFromNodeId(gpuNode);
#define BUFFER_SIZE (64*1024)
#define NPACKETS 1
#define COPY_SIZE (BUFFER_SIZE / NPACKETS)
HsaMemoryBuffer srcBuf(BUFFER_SIZE, 0, true);
HsaMemoryBuffer dstBuf(BUFFER_SIZE, gpuNode, false, hsakmt_is_dgpu() ? true : false);
SDMAQueue queue;
ASSERT_SUCCESS_GPU(queue.Create(gpuNode), gpuNode);
std::ostream &log = LOG();
char progress[] = "-\b";
log << "Running ... ";
for (unsigned i = 0; i < 100000; i++) {
if (i % 1000 == 0) {
const char progressSteps[4] = {'-', '\\', '|', '/'};
progress[0] = progressSteps[(i/1000) % 4];
log << progress;
}
for (unsigned j = 0; j < NPACKETS; j++)
queue.PlacePacket(
SDMACopyDataPacket(queue.GetFamilyId(), dstBuf.As<char *>()+COPY_SIZE*j,
srcBuf.As<char *>()+COPY_SIZE*j, COPY_SIZE));
queue.SubmitPacket();
/* Waste a variable amount of time. Submission timing
* while SDMA runs concurrently seems to be critical for
* reproducing the hang
*/
for (int k = 0; k < (i & 0xfff); k++)
memcpy(srcBuf.As<char *>()+PAGE_SIZE, srcBuf.As<char *>(), 1024);
/* Wait for idle every 8 packets to allow the SDMA engine to
* run concurrently for a bit without getting too far ahead
*/
if ((i & 0x7) == 0)
queue.Wait4PacketConsumption();
}
log << "Done." << std::endl;
queue.PlaceAndSubmitPacket(SDMAWriteDataPacket(queue.GetFamilyId(), srcBuf.As<unsigned *>(), 0x02020202));
queue.Wait4PacketConsumption();
EXPECT_TRUE_GPU(WaitOnValue(srcBuf.As<unsigned int*>(), 0x02020202), gpuNode);
EXPECT_SUCCESS_GPU(queue.Destroy(), gpuNode);
}
TEST_F(KFDQMTest, SdmaConcurrentCopies) {
TEST_START(TESTPROFILE_RUNALL)
ASSERT_SUCCESS(KFDTestLaunch([this](int gpuNode) {
this->SdmaConcurrentCopies(gpuNode);
}));
TEST_END
}
void KFDQMTest::DisableCpQueueByUpdateWithNullAddress(int gpuNode) {
HsaMemoryBuffer destBuf(PAGE_SIZE, gpuNode, false);
destBuf.Fill(0xFFFFFFFF);
PM4Queue queue;
ASSERT_SUCCESS_GPU(queue.Create(gpuNode), gpuNode);
HsaEvent *event;
ASSERT_SUCCESS_GPU(CreateQueueTypeEvent(false, false, gpuNode, &event), gpuNode);
queue.PlaceAndSubmitPacket(PM4WriteDataPacket(destBuf.As<unsigned int*>(), 0, 0));
queue.Wait4PacketConsumption(event);
WaitOnValue(destBuf.As<unsigned int*>(), 0);
destBuf.Fill(0xFFFFFFFF);
EXPECT_SUCCESS_GPU(queue.Update(BaseQueue::DEFAULT_QUEUE_PERCENTAGE, BaseQueue::DEFAULT_PRIORITY, true), gpuNode);
queue.PlaceAndSubmitPacket(PM4WriteDataPacket(destBuf.As<unsigned int*>(), 1, 1));
// Don't sync since we don't expect rptr to change when the queue is disabled.
Delay(2000);
EXPECT_EQ_GPU(destBuf.As<unsigned int*>()[0], 0xFFFFFFFF, gpuNode)
<< "Packet executed even though the queue is supposed to be disabled!";
EXPECT_SUCCESS_GPU(queue.Update(BaseQueue::DEFAULT_QUEUE_PERCENTAGE, BaseQueue::DEFAULT_PRIORITY, false), gpuNode);
queue.Wait4PacketConsumption(event);
WaitOnValue(destBuf.As<unsigned int*>(), 1);
hsaKmtDestroyEvent(event);
EXPECT_SUCCESS_GPU(queue.Destroy(), gpuNode);
}
TEST_F(KFDQMTest, DisableCpQueueByUpdateWithNullAddress) {
TEST_START(TESTPROFILE_RUNALL)
ASSERT_SUCCESS(KFDTestLaunch([this](int gpuNode) {
this->DisableCpQueueByUpdateWithNullAddress(gpuNode);
}));
TEST_END
}
void KFDQMTest::DisableSdmaQueueByUpdateWithNullAddress(int gpuNode) {
HsaMemoryBuffer destBuf(PAGE_SIZE, gpuNode, false);
destBuf.Fill(0xFFFFFFFF);
SDMAQueue queue;
ASSERT_SUCCESS_GPU(queue.Create(gpuNode), gpuNode);
queue.PlaceAndSubmitPacket(SDMAWriteDataPacket(queue.GetFamilyId(), destBuf.As<void*>(), 0));
WaitOnValue(destBuf.As<unsigned int*>(), 0);
destBuf.Fill(0xFFFFFFFF);
EXPECT_SUCCESS_GPU(queue.Update(BaseQueue::DEFAULT_QUEUE_PERCENTAGE, BaseQueue::DEFAULT_PRIORITY, true), gpuNode);
queue.PlaceAndSubmitPacket(SDMAWriteDataPacket(queue.GetFamilyId(), destBuf.As<void*>(), 0));
// Don't sync since we don't expect rptr to change when the queue is disabled.
Delay(2000);
EXPECT_EQ_GPU(destBuf.As<unsigned int*>()[0], 0xFFFFFFFF, gpuNode)
<< "Packet executed even though the queue is supposed to be disabled!";
EXPECT_SUCCESS_GPU(queue.Update(BaseQueue::DEFAULT_QUEUE_PERCENTAGE, BaseQueue::DEFAULT_PRIORITY, false), gpuNode);
queue.Wait4PacketConsumption();
WaitOnValue(destBuf.As<unsigned int*>(), 0);
EXPECT_SUCCESS_GPU(queue.Destroy(), gpuNode);
}
TEST_F(KFDQMTest, DisableSdmaQueueByUpdateWithNullAddress) {
TEST_START(TESTPROFILE_RUNALL)
ASSERT_SUCCESS(KFDTestLaunch([this](int gpuNode) {
this->DisableSdmaQueueByUpdateWithNullAddress(gpuNode);
}));
TEST_END
}
void KFDQMTest::DisableCpQueueByUpdateWithZeroPercentage(int gpuNode) {
HsaMemoryBuffer destBuf(PAGE_SIZE, gpuNode, false);
destBuf.Fill(0xFFFFFFFF);
PM4Queue queue;
ASSERT_SUCCESS_GPU(queue.Create(gpuNode), gpuNode);
HsaEvent *event;
ASSERT_SUCCESS_GPU(CreateQueueTypeEvent(false, false, gpuNode, &event), gpuNode);
PM4WriteDataPacket packet1, packet2;
packet1.InitPacket(destBuf.As<unsigned int*>(), 0, 0);
packet2.InitPacket(destBuf.As<unsigned int*>(), 1, 1);
queue.PlaceAndSubmitPacket(packet1);
queue.Wait4PacketConsumption(event);
WaitOnValue(destBuf.As<unsigned int*>(), 0);
destBuf.Fill(0xFFFFFFFF);
EXPECT_SUCCESS_GPU(queue.Update(0/*percentage*/, BaseQueue::DEFAULT_PRIORITY, false), gpuNode);
queue.PlaceAndSubmitPacket(packet2);
// Don't sync since we don't expect rptr to change when the queue is disabled.
Delay(2000);
EXPECT_EQ_GPU(destBuf.As<unsigned int*>()[0], 0xFFFFFFFF, gpuNode)
<< "Packet executed even though the queue is supposed to be disabled!";
EXPECT_SUCCESS_GPU(queue.Update(BaseQueue::DEFAULT_QUEUE_PERCENTAGE, BaseQueue::DEFAULT_PRIORITY, false), gpuNode);
queue.Wait4PacketConsumption(event);
WaitOnValue(destBuf.As<unsigned int*>(), 1);
hsaKmtDestroyEvent(event);
EXPECT_SUCCESS_GPU(queue.Destroy(), gpuNode);
}
TEST_F(KFDQMTest, DisableCpQueueByUpdateWithZeroPercentage) {
TEST_START(TESTPROFILE_RUNALL)
ASSERT_SUCCESS(KFDTestLaunch([this](int gpuNode) {
this->DisableCpQueueByUpdateWithZeroPercentage(gpuNode);
}));
TEST_END
}
void KFDQMTest::CreateQueueStressSingleThreaded(int gpuNode) {
static const HSAuint64 TEST_TIME_SEC = 15;
HSAuint64 initialTime = GetSystemTickCountInMicroSec();
unsigned int numIter = 0;
HSAuint64 timePassed = 0;
do {
// The following means we'll get the order 0,0 => 0,1 => 1,0 => 1,1 so we cover all options.
unsigned int firstToCreate = (numIter % 2 != 0) ? 1 : 0;
unsigned int firstToDestroy = (numIter % 4 > 1) ? 1 : 0;
unsigned int secondToCreate = (firstToCreate + 1)%2;
unsigned int secondToDestroy = (firstToDestroy + 1)%2;
BaseQueue *queues[2] = {new PM4Queue(), new SDMAQueue()};
ASSERT_SUCCESS_GPU(queues[firstToCreate]->Create(gpuNode), gpuNode);
ASSERT_SUCCESS_GPU(queues[secondToCreate]->Create(gpuNode),gpuNode);
EXPECT_SUCCESS_GPU(queues[firstToDestroy]->Destroy(), gpuNode);
EXPECT_SUCCESS_GPU(queues[secondToDestroy]->Destroy(), gpuNode);
delete queues[0];
delete queues[1];
++numIter;
HSAuint64 curTime = GetSystemTickCountInMicroSec();
timePassed = (curTime - initialTime) / 1000000;
} while (timePassed < TEST_TIME_SEC);
}
TEST_F(KFDQMTest, CreateQueueStressSingleThreaded) {
TEST_START(TESTPROFILE_RUNALL)
ASSERT_SUCCESS(KFDTestLaunch([this](int gpuNode) {
this->CreateQueueStressSingleThreaded(gpuNode);
}));
TEST_END
}
void KFDQMTest::OverSubscribeCpQueues(int gpuNode) {
const HSAuint32 m_FamilyId = GetFamilyIdFromNodeId(gpuNode);
if (m_FamilyId == FAMILY_CI || m_FamilyId == FAMILY_KV) {
LOG() << "Skipping test: CI doesn't have HW scheduling." << std::endl;
return;
}
/* The max queues per process is 1024 limited by
* KFD, so MAX_CP_QUEUES is needed to adapt it
* when total queues exceed it.
*/
static const unsigned int MAX_CP_QUEUES = g_TestGPUsNum > 15 ?
1024 / g_TestGPUsNum :
65;
static const unsigned int MAX_PACKETS = 100;
HsaMemoryBuffer destBuf(PAGE_SIZE, gpuNode, false);
destBuf.Fill(0xFF);
PM4Queue queues[MAX_CP_QUEUES];
for (unsigned int qidx = 0; qidx < MAX_CP_QUEUES; ++qidx)
ASSERT_SUCCESS_GPU(queues[qidx].Create(gpuNode), gpuNode) << " QueueId=" << qidx;
for (unsigned int qidx = 0; qidx < MAX_CP_QUEUES; ++qidx) {
unsigned int pktSizeDw = 0;
for (unsigned int i = 0; i < MAX_PACKETS; i++) {
PM4WriteDataPacket packet;
packet.InitPacket(destBuf.As<unsigned int*>()+qidx*2, qidx+i, qidx+i); // two dwords per packet
queues[qidx].PlacePacket(packet);
}
}
for (unsigned int qidx = 0; qidx < MAX_CP_QUEUES; ++qidx)
queues[qidx].SubmitPacket();
// Delaying for 5 seconds in order to get all the results
Delay(5000);
for (unsigned int qidx = 0; qidx < MAX_CP_QUEUES; ++qidx)
EXPECT_TRUE_GPU(queues[qidx].AllPacketsSubmitted(), gpuNode)<< "QueueId=" << qidx;;
for (unsigned int qidx = 0; qidx < MAX_CP_QUEUES; ++qidx)
EXPECT_SUCCESS_GPU(queues[qidx].Destroy(), gpuNode);
}
TEST_F(KFDQMTest, OverSubscribeCpQueues) {
TEST_START(TESTPROFILE_RUNALL)
ASSERT_SUCCESS(KFDTestLaunch([this](int gpuNode) {
this->OverSubscribeCpQueues(gpuNode);
}));
TEST_END
}
HSAint64 KFDQMTest::TimeConsumedwithCUMask(int node, uint32_t* mask, uint32_t mask_count) {
HsaMemoryBuffer isaBuffer(PAGE_SIZE, node, true/*zero*/, false/*local*/, true/*exec*/);
HsaMemoryBuffer dstBuffer(PAGE_SIZE, node, true, false, false);
HsaMemoryBuffer ctlBuffer(PAGE_SIZE, node, true, false, false);
EXPECT_SUCCESS(m_pAsm->RunAssembleBuf(LoopIsa, isaBuffer.As<char*>()));
Dispatch dispatch(isaBuffer);
dispatch.SetDim(1024, 16, 16);
PM4Queue queue;
EXPECT_SUCCESS(queue.Create(node));
EXPECT_SUCCESS(queue.SetCUMask(mask, mask_count));
queue.SetSkipWaitConsump(true);
HSAuint64 startTime = GetSystemTickCountInMicroSec();
dispatch.Submit(queue);
dispatch.Sync();
HSAuint64 endTime = GetSystemTickCountInMicroSec();
EXPECT_SUCCESS(queue.Destroy());
return endTime - startTime;
}
/* To cover for outliers, allow us to get the Average time based on a specified number of iterations */
HSAint64 KFDQMTest::GetAverageTimeConsumedwithCUMask(int node, uint32_t* mask, uint32_t mask_count, int iterations) {
HSAint64 timeArray[iterations];
HSAint64 timeTotal = 0;
if (iterations < 1) {
LOG() << "ERROR: At least 1 iteration must be performed" << std::endl;
return 0;
}
for (int x = 0; x < iterations; x++) {
timeArray[x] = TimeConsumedwithCUMask(node, mask, mask_count);
timeTotal += timeArray[x];
}
if (timeTotal == 0) {
LOG() << "ERROR: Total time reported as 0. Exiting" << std::endl;
return 0;
}
for (int x = 0; x < iterations; x++) {
HSAint64 variance = timeArray[x] / (timeTotal / iterations);
if (variance < CuNegVariance || variance > CuPosVariance)
LOG() << "WARNING: Measurement #" << x << "/" << iterations << " (" << timeArray[x]
<< ") is at least " << CuVariance*100 << "% away from the mean (" << timeTotal/iterations << ")"
<< std::endl;
}
return timeTotal / iterations;
}
/*
* Apply CU masking in a linear fashion, adding 1 CU per iteration
* until all Shader Engines are full
*/
void KFDQMTest::BasicCuMaskingLinear(int gpuNode) {
const HSAuint32 m_FamilyId = GetFamilyIdFromNodeId(gpuNode);
if (m_FamilyId >= FAMILY_VI) {
const HsaNodeProperties *pNodeProperties = Get_NodeInfo()->GetNodeProperties(gpuNode);
uint32_t ActiveCU = (pNodeProperties->NumFComputeCores / pNodeProperties->NumSIMDPerCU);
uint32_t numSEs = pNodeProperties->NumShaderBanks;
LOG() << std::dec << "# Compute cores: " << pNodeProperties->NumFComputeCores << std::endl;
LOG() << std::dec << "# SIMDs per CU: " << pNodeProperties->NumSIMDPerCU << std::endl;
LOG() << std::dec << "# Shader engines: " << numSEs << std::endl;
LOG() << std::dec << "# Active CUs: " << ActiveCU << std::endl;
HSAint64 TimewithCU1, TimewithCU;
uint32_t maskNumDwords = (ActiveCU + 31) / 32; /* Round up to the nearest multiple of 32 */
uint32_t maskNumBits = maskNumDwords * 32;
uint32_t mask[maskNumDwords];
double ratio;
mask[0] = 0x1;
for (int i = 1; i < maskNumDwords; i++)
mask[i] = 0x0;
/* Execute once to get any HW optimizations out of the way */
TimeConsumedwithCUMask(gpuNode, mask, maskNumBits);
LOG() << "Getting baseline performance numbers (CU Mask: 0x1)" << std::endl;
TimewithCU1 = GetAverageTimeConsumedwithCUMask(gpuNode, mask, maskNumBits, 3);
for (int nCUs = 2; nCUs <= ActiveCU; nCUs++) {
int maskIndex = (nCUs - 1) / 32;
mask[maskIndex] |= 1 << ((nCUs - 1) % 32);
TimewithCU = TimeConsumedwithCUMask(gpuNode, mask, maskNumBits);
ratio = (double)(TimewithCU1) / ((double)(TimewithCU) * nCUs);
LOG() << "Expected performance of " << nCUs << " CUs vs 1 CU:" << std::endl;
LOG() << std::setprecision(2) << CuNegVariance << " <= " << std::fixed << std::setprecision(8)
<< ratio << " <= " << std::setprecision(2) << CuPosVariance << std::endl;
EXPECT_TRUE((ratio >= CuNegVariance) && (ratio <= CuPosVariance));
RECORD(ratio) << "Ratio-" << nCUs << "-CUs";
}
} else {
LOG() << "Skipping test: Test not supported for family ID 0x" << m_FamilyId << "." << std::endl;
}
}
TEST_F(KFDQMTest, BasicCuMaskingLinear) {
TEST_START(TESTPROFILE_RUNALL);
ASSERT_SUCCESS(KFDTestLaunch([this](int gpuNode) {
this->BasicCuMaskingLinear(gpuNode);
}));
TEST_END
}
// ====== ExtendedCuMasking Helper Functions ====== //
#define CUMASK_DEBUG 0 // Enable extra output for debugging issues
#if CUMASK_DEBUG
#define DBG_PRINT printf
#else
#define DBG_PRINT
#endif
/*
* Helper function to print multi-dword mask.
*
* pHeader: A non-NULL pointer to a string to use as the header.
* pMask: A pointer to the mask to print out.
* numDwords: Number of elements in mask array.
*
*/
static void printMask(const char *pHeader, uint32_t *pMask, uint32_t numDwords) {
printf("%s0x", pHeader);
for (int i = numDwords - 1; i >= 0; i--) {
printf("%08x", pMask[i]);
}
printf("\n");
}
/*
* Set the CU mask for each specified WGPs.
*
* Note: The effect is cumulative, function can be called multiple times to
* set up additional WGPs in the provided pMask.
*
* pMask: A non-NULL pointer to the CU mask.
* maskConfig: Information on GPU configuration.
* seMask: Specifies SEs that are targetted.
* saMask: Specifies SAs that are targetted within the SEs specified.
* wgpMask: Specifies WGPs that are targetted within the (SE,SA) specified.
*
* For seMask, saMask, and wgpMask:
* One bit per SE/SA/WGP, multiple bits can be specified.
* Masks cannot be 0 (at least 1 SE, 1 SA and 1 WGP must be specified).
* Special value: -1 (specifies ALL)
*
*/
static bool setCUMask(uint32_t *pMask, mask_config_t maskConfig, uint32_t seMask, uint32_t saMask, uint32_t wgpMask) {
bool result = true;
if (pMask) {
if (seMask && saMask && wgpMask) { // proceed only with non-zero mask
for (int i = 0; i < maskConfig.numWGPperSA; i++) {
if (((wgpMask >> i) & 1)) {
for (int j = 0; j < maskConfig.numSAperSE; j++) {
if (((saMask >> j) & 1)) {
for (int k = 0; k < maskConfig.numSEs; k++) {
if (((seMask >> k) & 1)) {
uint32_t insLoc = k * 2 + j * (2 * maskConfig.numSEs) + i * (2 * maskConfig.numSEs * maskConfig.numSAperSE);
pMask[insLoc / 32] |= (0x3 << (insLoc % 32));
}
}
}
}
}
}
} else {
LOG() << "ERROR: SE/SA/WGP mask values must be non-zero!\n";
result = false;
}
} else {
LOG() << "ERROR: pMask is NULL!\n";
result = false;
}
return result;
}
/*
* Compute an adjusted CU mask to use when some WGPs are inactive.
*
* The adjusted mask takes into account the inactive WGPs by removing their corresponding
* bits from the mask as these are skipped by KFD. As bits are removed from the mask,
* the remaining bit values are shifted right.
*
* pAdjMask: A non-NULL pointer where the adjusted mask will be written.
* pMask: A non-NULL pointer to the CU mask.
* maskConfig: Information on GPU configuration.
*
* Returns:
* true: If adjusted mask has one or more non-zero bit set.
* false: If the adjusted mask is all zeroes (no WGPs left to do work).
*
* When false is returned, we should skipped the specific test scenario.
*
*/
bool adjustMask(uint32_t *pAdjMask, uint32_t *pMask, mask_config_t maskConfig) {
int wi = 0;
int totalBits = maskConfig.numBits;
bool nonZero = false;
uint32_t *tempInactiveMask = new uint32_t[maskConfig.numDwords]{};
uint32_t *tempAdjustMask = new uint32_t[maskConfig.numDwords]{};
/*
* KFD encodes all the active WGP at the lowest bits in MQD registers.
*
* If WGP3 is inactive in a SA, it will be encoded by KFD as: 0x3f.
* If WGP1 is inactive in a SA, it will be encoded by KFD also as: 0x3f.
*
* We need to adjust for that.
*
* For each SA, we need to "compress" all the active WGP settings together.
* i.e. if WGP1 is inactive, we need to put: (x WGP3 WGP2 WGP0) in our CU mask array.
*
* Note that WGPs in same SA are not encoded consecutively in the CU mask array, we need to take into account that.
*
* Once this step is done, we need to remove any "inactive" entries from the CU mask array as they are skipped by KFD.
*
*/
// Offset to get the next WGP in a SA.
// Ex: If WPG0 is at offset n, WGP1 is at offset n+nextWGPOffset in the CU mask array
const uint32_t nextWGPOffset = 2 * maskConfig.numSEs * maskConfig.numSAperSE;
for (int i = 0; i < maskConfig.numSEs; i++) {
for (int j = 0; j < maskConfig.numSAperSE; j++) {
// Location of WGP0 for (SE: i, SA: j)
uint32_t wgp0Loc = 2 * (j * maskConfig.numSEs + i);
// Location of last WGP
uint32_t wgpLastLoc = wgp0Loc + nextWGPOffset * (maskConfig.numWGPperSA - 1);
// Where to write the next active WGP
uint32_t activeWriteIndex = wgp0Loc;
// Where to write the next inactive WGP
uint32_t inactiveWriteIndex = wgpLastLoc;
/*
* Iterate over the WGPs for (SE: i, SA: j) and write all the active ones in tempAdjustMask, leaving the inactive ones.
* This condenses the WGPs for the (SE,SA) pair being processed.
*
* Generate temporary adjust mask with the inactive ones.
*
* At the end we have:
* tempAdjustMask: ( 0 WGP3 WGP2 WGP1 ) (2 bits per WGP)
* tempInactiveMask: ( 3 0 0 0 ) (2 bits per WGP)
*/
for (int k = wgp0Loc; k < totalBits; k += nextWGPOffset) {
if ((maskConfig.pInactiveMask[k / 32] & (0x3 << (k % 32))) != 0) {
tempInactiveMask[inactiveWriteIndex / 32] |= (0x3 << (inactiveWriteIndex % 32));
inactiveWriteIndex -= nextWGPOffset;
} else {
uint32_t value = (pMask[k / 32] >> (k % 32)) & 0x3;
uint32_t newValue = value << (activeWriteIndex % 32);
tempAdjustMask[activeWriteIndex / 32] |= newValue;
activeWriteIndex += nextWGPOffset;
}
}
}
}
// Now we remove all the inactive entries and generate the final adjusted mask.
memset(pAdjMask, 0, sizeof(uint32_t) * maskConfig.numDwords);
for (int ri = 0; ri < totalBits; ri += 2) {
uint32_t value = (tempAdjustMask[ri / 32] >> (ri % 32)) & 0x3;
if ((tempInactiveMask[ri / 32] & (0x3 << (ri % 32))) != 0)
{
// skip that entry
}
else
{
uint32_t newValue = value << (wi % 32);
pAdjMask[wi / 32] |= newValue;
wi += 2;
if (value != 0) {
nonZero = true;
}
}
}
#if CUMASK_DEBUG
printf("\nAdjusting mask:\n");
printMask(" mask: ", pMask, maskConfig.numDwords);
printMask(" inactive: ", maskConfig.pInactiveMask, maskConfig.numDwords);
printMask("temp adjusted: ", tempAdjustMask, maskConfig.numDwords);
printMask("temp inactive: ", tempInactiveMask, maskConfig.numDwords);
printMask(" adjusted: ", pAdjMask, maskConfig.numDwords);
printf("\n");
#endif //CUMASK_DEBUG
delete[] tempInactiveMask;
delete[] tempAdjustMask;
return nonZero;
}
/*
* Validates the result of a test.
*
* pMask: A non-NULL pointer to the CU mask that was used for the test.
* maskConfig: Information on GPU configuration.
* numWorkItems: Number of work items used for shader execution.
* pOutput: Pointer to the output array.
* pResultMask: If non-NULL, result mask constructed from output is stored at that memory location.
*
*/
static bool validateTest(uint32_t *pMask, mask_config_t maskConfig, uint32_t numWorkItems, out_data_t *pOutput, uint32_t *pResultMask)
{
uint32_t resultMask[maskConfig.numDwords];
bool result = false;
memset(resultMask, 0, sizeof(resultMask));
for (int i = 0; i < numWorkItems; i++) {
DBG_PRINT("=== % 4d: 0x%08x [ se: %2d, sa: %2d, wgp: %2d]\n", i, pOutput[i].data, pOutput[i].se, pOutput[i].sa, pOutput[i].wgp);
setCUMask(resultMask, maskConfig,
1 << pOutput[i].se,
1 << pOutput[i].sa,
1 << pOutput[i].wgp);
}
if (pResultMask) {
memcpy(pResultMask, resultMask, sizeof(resultMask));
}
if (maskConfig.pInactiveMask) {
// If some WGPs were inactive, compute a verify mask taking into account the inactive WGPs.
uint32_t verifyMask[maskConfig.numDwords];
memset(verifyMask, 0, sizeof(verifyMask));
for (int i = 0; i < maskConfig.numDwords; i++) {
verifyMask[i] = pMask[i] & ~maskConfig.pInactiveMask[i];
}
#if CUMASK_DEBUG
printf("\nValidate test:\n");
printMask(" mask: ", pMask, maskConfig.numDwords);
printMask(" resultMask: ", resultMask, maskConfig.numDwords);
printMask("inactiveMask: ", maskConfig.pInactiveMask, maskConfig.numDwords);
printMask(" verifyMask: ", verifyMask, maskConfig.numDwords);
#endif //CUMASK_DEBUG
result = (memcmp(verifyMask, resultMask, sizeof(resultMask)) == 0);
} else {
#if CUMASK_DEBUG
printf("\nValidate test:\n");
printMask(" mask: ", pMask, maskConfig.numDwords);
printMask(" resultMask: ", resultMask, maskConfig.numDwords);
#endif //CUMASK_DEBUG
result = (memcmp(pMask, resultMask, sizeof(resultMask)) == 0);
}
DBG_PRINT(" Result: %s\n\n", result ? "PASS" : "FAIL");
return result;
}
/*
* Set CU Mask, submit the testing shader, and validate the results.
*
* gpuNode: The node to use for the test.
* pMask: A non-NULL pointer to the CU mask to use for the test.
* maskConfig: Information on GPU configuration.
* programBuffer: The buffer that contains the shader program.
* numWorkItems: The number of work items to use.
* pOutput: A non-NULL pointer to the output buffer used by the shader.
* pResultMask: If non-NULL, result mask constructed from output is stored at that memory location.
*
*/
static bool testCUMask(int gpuNode, uint32_t *pMask, mask_config_t maskConfig, HsaMemoryBuffer &programBuffer, uint32_t numWorkItems, out_data_t *pOutput, uint32_t *pResultMask = NULL) {
PM4Queue queue;
uint32_t *pAdjMask = NULL;
uint32_t adjMask[maskConfig.numDwords];
if (maskConfig.pInactiveMask) {
if (adjustMask(adjMask, pMask, maskConfig)) {
pAdjMask = adjMask;
} else {
// Adjusted mask is all zeroes, skip test and mark as passing.
return true;
}
} else {
pAdjMask = pMask;
}
Dispatch dispatch(programBuffer);
dispatch.SetArgs(NULL, pOutput);
dispatch.SetDim(numWorkItems, 1, 1);
EXPECT_SUCCESS_GPU(queue.Create(gpuNode), gpuNode);
EXPECT_SUCCESS_GPU(queue.SetCUMask(pAdjMask, maskConfig.numBits), gpuNode);
dispatch.Submit(queue);
dispatch.Sync();
EXPECT_SUCCESS_GPU(queue.Destroy(), gpuNode);
return validateTest(pMask, maskConfig, numWorkItems, pOutput, pResultMask);
}
/*
* ExtendedCuMasking
*
* Newer implementation of CU mask testing that focuses on correctness of masking.
*
* Unlike previous implementations, this new implementation does not rely on performance
* measurements to decide if the masking took place. Instead, this implementation checks
* if waves were executed on all the CUs enabled and only the CUs enabled.
*
* Implementation does a series of tests, new tests can be easily added as needed.
*
* For each test, these steps are performed:
*
* 1) Decide the units that are enabled for the test (SEs, SAs, WGPs).
* 2) Generate a CU mask that specifies the WGPs enabled on each (SE,SA) pairs.
* 3) Set the mask for the queue and run a special shader.
* 4) Shader records in a buffer the unit that is used by the wave (SE,SA,WGP).
* 5) Test program analyses the results and verifies if shader used all and only the
* WGP units specified by the mask.
*
* Multiple tests are done with different combinations.
* There are (2^numWGPs - 1) possibilities, not everything can be tested.
*
* For each new ASIC supported, the following changes might be required:
* 1) Minor shader changes to put fill information into buffer.
* 2) Format of out_data_t struct.
* 3) Changes to validation code.
*
*/
void KFDQMTest::extendedCuMasking(int gpuNode) {
const HSAuint32 m_FamilyId = GetFamilyIdFromNodeId(gpuNode);
if (m_FamilyId >= FAMILY_GFX12) { // Supporting GFX12 and up for now
// Lock to prevent interleave of logging on multigpu (multithreaded) testing
static std::mutex logMutex;
const HsaNodeProperties *pProps = Get_NodeInfo()->GetNodeProperties(gpuNode);
const uint32_t activeCU = (pProps->NumFComputeCores / pProps->NumSIMDPerCU);
const uint32_t numSEs = pProps->NumShaderBanks;
const uint32_t numSAperSE = pProps->NumArrays;
const uint32_t numWGPperSA = pProps->NumCUPerArray / 2;
const uint32_t maxCU = numSEs * numSAperSE * numWGPperSA * 2;
std::ostringstream nodeStream;
nodeStream << "(Node " << gpuNode << ")";
const std::string nodeStr = nodeStream.str();
logMutex.lock();
LOG() << std::endl;
LOG() << std::dec << "****** GFX Configuration " << nodeStr << " ******" << std::endl;
LOG() << std::dec << " Compute Cores (SIMD): " << std::setw(3) << pProps->NumFComputeCores << std::endl;
LOG() << std::dec << " SIMDs per CU: " << std::setw(3) << pProps->NumSIMDPerCU << std::endl;
LOG() << std::dec << " Active CUs: " << std::setw(3) << activeCU << std::endl;
LOG() << std::dec << " Max CUs: " << std::setw(3) << maxCU << std::endl;
LOG() << std::dec << " Shader Engines: " << std::setw(3) << numSEs << std::endl;
LOG() << std::dec << " SAs per SE: " << std::setw(3) << numSAperSE << std::endl;
LOG() << std::dec << " WGPs per SA: " << std::setw(3) << numWGPperSA << std::endl;
LOG() << std::dec << "****************************************" << std::endl;
logMutex.unlock();
const uint32_t maskNumDwords = (maxCU + 31) / 32; /* Round up to the nearest multiple of 32 */
const uint32_t maskNumBits = maskNumDwords * 32;
uint32_t mask[maskNumDwords];
uint32_t inactiveMask[maskNumDwords];
mask_config_t maskConfig = { maskNumDwords, maskNumBits, numSEs, numSAperSE, numWGPperSA, NULL };
/*
* Note: On system with WGPs, CU bits in the same WGP must be either both set or both unset
* i.e. enabling/disabling is on a per-WGP basis.
*
* Format of CU Mask array (Assuming 4 SEs)
*
* Bit Value Masking
*
* 0,1 0x03 SE0 SA0 WGP0 (i.e. CU0 and CU1)
* 2,3 0x0c SE1 SA0 WGP0
* 4,5 0x30 SE2 SA0 WGP0
* 6,7 0xc0 SE3 SA0 WGP0
*
* 8,9 0x0300 SE0 SA1 WGP0
* 10,11 0x0c00 SE1 SA1 WGP0
* 12,13 0x3000 SE2 SA1 WGP0
* 14,15 0xc000 SE3 SA1 WGP0
*
* 16,17 0x030000 SE0 SA0 WGP1
* 18,19 0x030000 SE1 SA0 WGP1
* ...
* 32,33 SE0 SA0 WGP2
* ...
* 48,49 SE0 SA0 WGP3
* ...
*
*/
/*
* Number of work items needs to be sufficiently large to have enough work items for each WGP enabled.
*
* Using total number of WGPs multiplied by 16.
*
*/
const uint32_t numWorkItems = 16 * numSEs * numSAperSE * numWGPperSA;
// Allocate buffers for program and output
HsaMemoryBuffer programBuffer(PAGE_SIZE, gpuNode, true, false, true);
HsaMemoryBuffer outputBuffer(((sizeof(out_data_t) * numWorkItems) + (PAGE_SIZE - 1)) & ~(PAGE_SIZE - 1), gpuNode, true, false, false);
out_data_t *pOutput = outputBuffer.As<out_data_t *>();
// Assemble shader
Assembler *pAsm = GetAssemblerFromNodeId(gpuNode);
ASSERT_NOTNULL_GPU(pAsm, gpuNode);
ASSERT_SUCCESS_GPU(pAsm->RunAssembleBuf(CheckCuMaskIsa, programBuffer.As<char*>()), gpuNode);
/*
* Check and record any inactive WPGs.
*
*/
memset(mask, 0, sizeof(mask));
memset(inactiveMask, 0, sizeof(inactiveMask));
// Use full mask and collect all active CUs in inactiveMask
setCUMask(mask, maskConfig, -1, -1, -1);
if (testCUMask(gpuNode, mask, maskConfig, programBuffer, numWorkItems, pOutput, inactiveMask)) {
// Using full mask, if all CUs are used, we expect them to be all active.
EXPECT_TRUE_GPU(activeCU == maxCU, gpuNode);
} else {
// Some CUs were not used, generate inactive mask and count inactive CUs.
uint32_t inactiveCount = 0;
// Flip bits and count inactive
for (int i = 0; i < maskNumDwords; i++) {
inactiveMask[i] = ~inactiveMask[i];
inactiveCount += __builtin_popcount(inactiveMask[i]);
}
// Check if what we detected is consistent with info from KFD
EXPECT_TRUE_GPU((activeCU + inactiveCount) == maxCU, gpuNode);
maskConfig.pInactiveMask = inactiveMask;
std::ostringstream logStr;
logStr << nodeStr << " Inactive CUs detected: " << inactiveCount << " 0x" << std::hex << std::setfill('0');
for (int i = maskNumDwords - 1; i >= 0; i--) {
logStr << std::setw(8) << inactiveMask[i];
}
LOG() << logStr.str() << std::endl;
}
/*
* Generate symmetric test configuration for all (SE, SA, WGP) combinations, one level at a time.
*
* Other levels fully enabled.
*
* Example: If testing SE disablement, all SA/WGP are enabled on the SE that are used.
* If testing SA disablement, all SE are used, all WGP are enabled on the SA enabled.
*
*/
uint32_t totalConfigTested = 0;
// All SE combination (0 not allowed, need at least one enabled)
LOG() << nodeStr << " === Testing SE mask (" << ((1 << numSEs) - 1) << " configs)\n";
for (int i = 1; i < (1 << numSEs); i++) {
memset(mask, 0, sizeof(mask));
DBG_PRINT("SE mask: 0x%x\n", i);
setCUMask(mask, maskConfig, i, -1, -1);
EXPECT_TRUE_GPU(testCUMask(gpuNode, mask, maskConfig, programBuffer, numWorkItems, pOutput), gpuNode);
totalConfigTested++;
}
// All SA combinations (0 not allowed, need at least one enabled)
LOG() << nodeStr << " === Testing SA mask (" << ((1 << numSAperSE) - 1) << " configs)\n";
for (uint32_t i = 1; i < (1 << numSAperSE); i++) {
memset(mask, 0, sizeof(mask));
DBG_PRINT("SA mask: 0x%x\n", i);
setCUMask(mask, maskConfig, -1, i, -1);
EXPECT_TRUE_GPU(testCUMask(gpuNode, mask, maskConfig, programBuffer, numWorkItems, pOutput), gpuNode);
totalConfigTested++;
}
// All WGP combinations (0 not allowed, need at least one enabled)
LOG() << nodeStr << " === Testing WGP mask (" << ((1 << numWGPperSA) - 1) << " configs)\n";
for (uint32_t i = 1; i < (1 << numWGPperSA); i++) {
memset(mask, 0, sizeof(mask));
DBG_PRINT("WGP mask: 0x%x\n", i);
setCUMask(mask, maskConfig, -1, -1, i);
EXPECT_TRUE_GPU(testCUMask(gpuNode, mask, maskConfig, programBuffer, numWorkItems, pOutput), gpuNode);
totalConfigTested++;
}
/*
* Linear Masking
*
* Enable one WGP at a time until they are all enabled.
*
*/
{
uint32_t totalWGPs = numSEs * numSAperSE * numWGPperSA;
LOG() << nodeStr << " === Testing linear mask (" << totalWGPs << " configs)\n";
memset(mask, 0, sizeof(mask));
for (int32_t i = 0; i < totalWGPs; i++) {
mask[i / 16] |= (0x3 << (i * 2));
#if CUMASK_DEBUG
printMask(" linear mask: ", mask, maskNumDwords);
#endif //CUMASK_DEBUG
EXPECT_TRUE_GPU(testCUMask(gpuNode, mask, maskConfig, programBuffer, numWorkItems, pOutput), gpuNode);
totalConfigTested++;
}
}
/*
* Random asymmetric config.
*
* Asymmetric, different WGPs/SAs are enabled/disabled on different SEs.
*
*/
{
uint32_t randomCount = 1000; // Total number of random test to perform
uint32_t seed = 1; // Specifying a seed to have deterministic random sequence
srand(seed);
LOG() << nodeStr << " === Testing " << randomCount << " random mask config...\n";
for (uint32_t i = 0; i < randomCount; i++) {
memset(mask, 0, sizeof(mask));
uint32_t wgpLeft = maxCU / 2; // init to total WGPs
uint32_t maskIndex = 0;
while (wgpLeft > 0) {
uint32_t wgpBlock = (wgpLeft > 16) ? 16 : wgpLeft; // max 16 WGPs at a time
wgpLeft -= wgpBlock;
/*
* Pick random number between 0 to (2^wgpBlock - 1) - 1.
* Then add 1 to get random number between 1 to (2^wgpBlock - 1).
* This ensure that we don't end up with 0 for all the dwords in the mask.
*/
uint32_t wgpMask = (rand() % ((1ULL << wgpBlock) - 1)) + 1;
// expand WGP mask to CU mask by doubling each individual bits.
uint32_t expandToCUMask = 0;
for (uint32_t j = 0; j < wgpBlock; j++) {
if (wgpMask & (1 << j)) {
expandToCUMask |= (0x3ULL << (j * 2));
}
}
DBG_PRINT("maskIndex: %u fullWGPMask: 0x%08x expand: 0x%08x\n", maskIndex, wgpMask, expandToCUMask);
mask[maskIndex++] = expandToCUMask;
}
EXPECT_TRUE_GPU(testCUMask(gpuNode, mask, maskConfig, programBuffer, numWorkItems, pOutput), gpuNode);
totalConfigTested++;
}
}
LOG() << std::endl;
LOG() << nodeStr << " Total config tested: " << totalConfigTested << std::endl;
LOG() << std::endl;
} else {
LOG() << "Skipping test: Test not supported for family ID 0x" << m_FamilyId << "." << std::endl;
}
}
TEST_F(KFDQMTest, ExtendedCuMasking) {
TEST_START(TESTPROFILE_RUNALL);
ASSERT_SUCCESS(KFDTestLaunch([this](int gpuNode) {
this->extendedCuMasking(gpuNode);
}));
TEST_END
}
#undef CUMASK_DEBUG
#undef DBG_PRINT
// ====== End of ExtendedCUMasking Functions ====== //
/**
* Apply CU masking where the number of CUs is equal across all Shader Engines
* This will work due to the HW splitting the workload unevenly across the Shader
* Engines when ((#ofCUs)/(#ofShaderEngines)) is not a whole number. The tests above
* will not yield viable results when an uneven distribution of CUs is used over multiple
* shader engines (e.g. 0x1000100030003), until the HW changes how it schedules work.
*/
void KFDQMTest::BasicCuMaskingEven(int gpuNode) {
const HSAuint32 m_FamilyId = GetFamilyIdFromNodeId(gpuNode);
if (m_FamilyId >= FAMILY_VI) {
const HsaNodeProperties *pNodeProperties = Get_NodeInfo()->GetNodeProperties(gpuNode);
uint32_t ActiveCU = (pNodeProperties->NumFComputeCores / pNodeProperties->NumSIMDPerCU);
uint32_t numShaderEngines = pNodeProperties->NumShaderBanks;
if (numShaderEngines == 1) {
LOG() << "Skipping test: Only 1 Shader Engine present." << std::endl;
return;
}
LOG() << std::dec << "# Compute cores: " << pNodeProperties->NumFComputeCores << std::endl;
LOG() << std::dec << "# SIMDs per CU: " << pNodeProperties->NumSIMDPerCU << std::endl;
LOG() << std::dec << "# Shader engines: " << numShaderEngines << std::endl;
LOG() << std::dec << "# Active CUs: " << ActiveCU << std::endl;
HSAint64 TimewithCU1, TimewithCU;
uint32_t maskNumDwords = (ActiveCU + 31) / 32; /* Round up to the nearest multiple of 32 */
uint32_t maskNumBits = maskNumDwords * 32;
uint32_t mask[maskNumDwords];
int numCuPerShader = ActiveCU / numShaderEngines;
double ratio;
/* In KFD we symmetrically map mask to all SEs:
* mask[0] bit0 -> se0 cu0;
* mask[0] bit1 -> se1 cu0;
* ... (if # SE is 4)
* mask[0] bit4 -> se0 cu1;
* ...
*/
/* Set Mask to 1 CU per SE */
memset(mask, 0, maskNumDwords * sizeof(uint32_t));
for (int i = 0; i < numShaderEngines; i++) {
int maskIndex = (i / 32) % maskNumDwords;
mask[maskIndex] |= 1 << (i % 32);
}
/* Execute once to get any HW optimizations out of the way */
TimeConsumedwithCUMask(gpuNode, mask, maskNumBits);
LOG() << "Getting baseline performance numbers (1 CU per SE)" << std::endl;
TimewithCU1 = GetAverageTimeConsumedwithCUMask(gpuNode, mask, maskNumBits, 3);
/* Each loop will add 1 more CU per SE. We use the mod and divide to handle
* when SEs aren't distributed in multiples of 32 (e.g. Tonga)
* OR the new bit in for simplicity instead of re-creating the mask each iteration
*/
for (int x = 0; x < numCuPerShader; x++) {
for (int se = 0; se < numShaderEngines; se++) {
int offset = x * numShaderEngines + se;
int maskIndex = (offset / 32) % maskNumDwords;
mask[maskIndex] |= 1 << (offset % 32);
}
int nCUs = x + 1;
TimewithCU = TimeConsumedwithCUMask(gpuNode, mask, maskNumBits);
ratio = (double)(TimewithCU1) / ((double)(TimewithCU) * nCUs);
LOG() << "Expected performance of " << nCUs << " CU(s)/SE vs 1 CU/SE:" << std::endl;
LOG() << std::setprecision(2) << CuNegVariance << " <= " << std::fixed << std::setprecision(8)
<< ratio << " <= " << std::setprecision(2) << CuPosVariance << std::endl;
EXPECT_TRUE_GPU((ratio >= CuNegVariance) && (ratio <= CuPosVariance), gpuNode);
RECORD(ratio) << "Ratio-" << nCUs << "-CUs";
}
} else {
LOG() << "Skipping test: Test not supported for family ID 0x" << m_FamilyId << "." << std::endl;
}
}
TEST_F(KFDQMTest, BasicCuMaskingEven) {
TEST_START(TESTPROFILE_RUNALL);
ASSERT_SUCCESS(KFDTestLaunch([this](int gpuNode) {
this->BasicCuMaskingEven(gpuNode);
}));
TEST_END
}
void KFDQMTest::testQueuePriority(int gpuNode, bool isSamePipe)
{
const HSAuint32 m_FamilyId = GetFamilyIdFromNodeId(gpuNode);
Assembler* m_pAsm;
m_pAsm = GetAssemblerFromNodeId(gpuNode);
ASSERT_NOTNULL_GPU(m_pAsm, gpuNode);
if (m_FamilyId < FAMILY_VI) {
LOG() << "Skipping test: Shader won't run on CI." << std::endl;
return;
}
// Reduce test case if running on emulator
// Reduction applies to all 3 dims (effect is cubic)
const int scaleDown = (g_IsEmuMode ? 4 : 1);
HsaMemoryBuffer syncBuf(PAGE_SIZE, gpuNode, true/*zero*/, false/*local*/, true/*exec*/);
HSAint32 *syncBuffer = syncBuf.As<HSAint32*>();
HsaMemoryBuffer isaBuffer(PAGE_SIZE, gpuNode, true/*zero*/, false/*local*/, true/*exec*/);
//ASSERT_SUCCESS(m_pAsm->RunAssembleBuf(LoopIsa, isaBuffer.As<char*>()));
ASSERT_SUCCESS_GPU(m_pAsm->RunAssembleBuf(LoopIsa, isaBuffer.As<char*>()), gpuNode);
Dispatch dispatch[2] = {
Dispatch(isaBuffer, true),
Dispatch(isaBuffer, true)
};
const int queueCount = isSamePipe ? 13 : 2;
int activeTaskBitmap = 0x3;
HSAuint64 startTime, endTime[2];
HsaEvent *pHsaEvent[2];
int numEvent = 2;
PM4Queue queue[queueCount];
HSA_QUEUE_PRIORITY priority[2] = {
HSA_QUEUE_PRIORITY_LOW,
HSA_QUEUE_PRIORITY_HIGH
};
int i;
/*
* For different pipe variation:
* Only two queues are created, they should be on two different pipes.
*
* For same pipe variation:
* queue[2..12] are dummy queues. Create queue in this sequence to
* render queue[0] and queue[1] on same pipe with no assumptions
* about the number of pipes used by KFD. Queue #12 is a multiple
* of 1, 2, 3 and 4, so it falls on pipe 0 for any number of pipes
*/
EXPECT_SUCCESS_GPU(queue[0].Create(gpuNode), gpuNode); // Queue 0 is on Pipe 0
if (isSamePipe) {
for (i = 2; i < queueCount; i++)
EXPECT_SUCCESS_GPU(queue[i].Create(gpuNode), gpuNode);
}
EXPECT_SUCCESS_GPU(queue[1].Create(gpuNode), gpuNode);
for (i = 0; i < 2; i++) {
syncBuffer[i] = -1;
queue[i].Update(BaseQueue::DEFAULT_QUEUE_PERCENTAGE, priority[i], false);
pHsaEvent[i] = dispatch[i].GetHsaEvent();
pHsaEvent[i]->EventData.EventData.SyncVar.SyncVar.UserData = &syncBuffer[i];
dispatch[i].SetDim(1024 / scaleDown , 16 / scaleDown, 16 / scaleDown);
}
startTime = GetSystemTickCountInMicroSec();
for (i = 0; i < 2; i++)
dispatch[i].Submit(queue[i]);
while (activeTaskBitmap > 0) {
hsaKmtWaitOnMultipleEvents(pHsaEvent, numEvent, false, g_TestTimeOut);
for (i = 0; i < 2; i++) {
if ((activeTaskBitmap & (1 << i)) && (syncBuffer[i] == pHsaEvent[i]->EventId)) {
endTime[i] = GetSystemTickCountInMicroSec();
activeTaskBitmap &= ~(1 << i);
}
}
}
for (i = 0; i < 2; i++) {
int usecs = endTime[i] - startTime;
LOG() << "Task priority: " << std::dec << priority[i] << "\t";
LOG() << "Task duration: " << std::dec << std::setw(10) << usecs << " usecs" << std::endl;
}
for (i = 0; i < queueCount; i++) {
EXPECT_SUCCESS_GPU(queue[i].Destroy(), gpuNode);
}
}
void KFDQMTest::QueuePriorityOnDifferentPipe(int gpuNode) {
testQueuePriority(gpuNode, false);
}
TEST_F(KFDQMTest, QueuePriorityOnDifferentPipe) {
TEST_START(TESTPROFILE_RUNALL);
ASSERT_SUCCESS(KFDTestLaunch([this](int gpuNode) {
this->QueuePriorityOnDifferentPipe(gpuNode);
}));
TEST_END
}
void KFDQMTest::QueuePriorityOnSamePipe(int gpuNode) {
testQueuePriority(gpuNode, true);
}
TEST_F(KFDQMTest, QueuePriorityOnSamePipe) {
TEST_START(TESTPROFILE_RUNALL);
ASSERT_SUCCESS(KFDTestLaunch([this](int gpuNode) {
this->QueuePriorityOnSamePipe(gpuNode);
}));
TEST_END
}
void KFDQMTest::SyncDispatch(const HsaMemoryBuffer& isaBuffer, void* arg0, void* arg1, int node) {
PM4Queue queue;
if (node == -1)
node = m_NodeInfo.HsaDefaultGPUNode();
ASSERT_GE_GPU(node, 0, node) << "failed to get GPU Node";
Dispatch dispatch(isaBuffer);
dispatch.SetArgs(arg0, arg1);
dispatch.SetDim(1, 1, 1);
ASSERT_SUCCESS_GPU(queue.Create(node), node);
dispatch.Submit(queue);
dispatch.Sync();
EXPECT_SUCCESS_GPU(queue.Destroy(), node);
}
void KFDQMTest::EmptyDispatch(int gpuNode) {
Assembler* m_pAsm;
m_pAsm = GetAssemblerFromNodeId(gpuNode);
ASSERT_NOTNULL_GPU(m_pAsm, gpuNode);
HsaMemoryBuffer isaBuffer(PAGE_SIZE, gpuNode, true/*zero*/, false/*local*/, true/*exec*/);
ASSERT_SUCCESS_GPU(m_pAsm->RunAssembleBuf(NoopIsa, isaBuffer.As<char*>()), gpuNode);
SyncDispatch(isaBuffer, NULL, NULL, gpuNode);
}
TEST_F(KFDQMTest, EmptyDispatch) {
TEST_START(TESTPROFILE_RUNALL);
ASSERT_SUCCESS(KFDTestLaunch([this](int gpuNode) {
this->EmptyDispatch(gpuNode);
}));
TEST_END
}
void KFDQMTest::SimpleWriteDispatch(int gpuNode) {
Assembler* m_pAsm;
m_pAsm = GetAssemblerFromNodeId(gpuNode);
ASSERT_NOTNULL_GPU(m_pAsm, gpuNode);
HsaMemoryBuffer isaBuffer(PAGE_SIZE, gpuNode, true/*zero*/, false/*local*/, true/*exec*/);
HsaMemoryBuffer srcBuffer(PAGE_SIZE, gpuNode, false);
HsaMemoryBuffer destBuffer(PAGE_SIZE, gpuNode);
srcBuffer.Fill(FILL_VALUE);
ASSERT_SUCCESS_GPU(m_pAsm->RunAssembleBuf(CopyDwordIsa, isaBuffer.As<char*>()),gpuNode);
SyncDispatch(isaBuffer, srcBuffer.As<void*>(), destBuffer.As<void*>(), gpuNode);
EXPECT_EQ(destBuffer.As<unsigned int*>()[0], FILL_VALUE);
}
TEST_F(KFDQMTest, SimpleWriteDispatch) {
TEST_START(TESTPROFILE_RUNALL);
ASSERT_SUCCESS(KFDTestLaunch([this](int gpuNode) {
this->SimpleWriteDispatch(gpuNode);
}));
TEST_END
}
void KFDQMTest::GpuMemCopyTest(int gpuNode) {
HSAuint32 m_FamilyId = GetFamilyIdFromNodeId(gpuNode);
if (m_FamilyId < FAMILY_AR) {
LOG() << "Skipping test: MultipleWordsDispatch test not yet available for this family id." << std::endl;
return;
}
const size_t bufSize = PAGE_SIZE;
HsaMemoryBuffer srcBuffer(bufSize, gpuNode, false);
HsaMemoryBuffer dstBuffer(bufSize, gpuNode, false);
HsaMemoryBuffer dstLocalBuffer(bufSize, gpuNode, false, true);
HsaMemoryBuffer verifyBuffer(bufSize, gpuNode, false);
srcBuffer.Fill(FILL_VALUE, 0, bufSize);
// SDMA copy
dstBuffer.Fill(INIT_VALUE, 0, bufSize);
ASSERT_TRUE(GPUMemCopy(dstBuffer.As<void*>(), srcBuffer.As<void*>(), bufSize, gpuNode, true));
for (size_t i = 0; i < bufSize / sizeof(unsigned int); ++i)
EXPECT_EQ(dstBuffer.As<unsigned int*>()[i], FILL_VALUE);
// Blit kernel copy
dstBuffer.Fill(INIT_VALUE, 0, bufSize);
ASSERT_TRUE(GPUMemCopy(dstBuffer.As<void*>(), srcBuffer.As<void*>(), bufSize, gpuNode, false));
for (size_t i = 0; i < bufSize / sizeof(unsigned int); ++i)
EXPECT_EQ(dstBuffer.As<unsigned int*>()[i], FILL_VALUE);
// SDMA copy to local memory
verifyBuffer.Fill(INIT_VALUE, 0, bufSize);
ASSERT_TRUE(GPUMemCopy(dstLocalBuffer.As<void*>(), srcBuffer.As<void*>(), bufSize, gpuNode, true));
ASSERT_TRUE(GPUMemCopy(verifyBuffer.As<void*>(), dstLocalBuffer.As<void*>(), bufSize, gpuNode, true));
for (size_t i = 0; i < bufSize / sizeof(unsigned int); ++i)
EXPECT_EQ(verifyBuffer.As<unsigned int*>()[i], FILL_VALUE);
// Blit kernel copy to local memory
verifyBuffer.Fill(INIT_VALUE, 0, bufSize);
ASSERT_TRUE(GPUMemCopy(dstLocalBuffer.As<void*>(), srcBuffer.As<void*>(), bufSize, gpuNode, false));
ASSERT_TRUE(GPUMemCopy(verifyBuffer.As<void*>(), dstLocalBuffer.As<void*>(), bufSize, gpuNode, false));
for (size_t i = 0; i < bufSize / sizeof(unsigned int); ++i)
EXPECT_EQ(verifyBuffer.As<unsigned int*>()[i], FILL_VALUE);
}
TEST_F(KFDQMTest, GpuMemCopyTest) {
TEST_START(TESTPROFILE_RUNALL);
ASSERT_SUCCESS(KFDTestLaunch([this](int gpuNode) {
this->GpuMemCopyTest(gpuNode);
}));
TEST_END;
}
void KFDQMTest::MultipleCpQueuesStressDispatch(int gpuNode) {
Assembler* m_pAsm;
m_pAsm = GetAssemblerFromNodeId(gpuNode);
ASSERT_NOTNULL_GPU(m_pAsm, gpuNode);
static const unsigned int MAX_CP_QUEUES = 16;
HsaMemoryBuffer isaBuffer(PAGE_SIZE, gpuNode, true/*zero*/, false/*local*/, true/*exec*/);
HsaMemoryBuffer srcBuffer(PAGE_SIZE, gpuNode, false);
HsaMemoryBuffer destBuffer(PAGE_SIZE, gpuNode);
unsigned int* src = srcBuffer.As<unsigned int*>();
unsigned int* dst = destBuffer.As<unsigned int*>();
static const HSAuint64 TEST_TIME_SEC = 15;
HSAuint64 initialTime, curTime;
unsigned int numIter = 0;
HSAuint64 timePassed = 0;
unsigned int i;
PM4Queue queues[MAX_CP_QUEUES];
Dispatch* dispatch[MAX_CP_QUEUES];
destBuffer.Fill(0xFF);
ASSERT_SUCCESS_GPU(m_pAsm->RunAssembleBuf(CopyDwordIsa, isaBuffer.As<char*>()), gpuNode);
for (i = 0; i < MAX_CP_QUEUES; ++i)
ASSERT_SUCCESS_GPU(queues[i].Create(gpuNode), gpuNode) << " QueueId=" << i;
initialTime = GetSystemTickCountInMicroSec();
do {
for (i = 0; i < MAX_CP_QUEUES; ++i) {
dispatch[i] = new Dispatch(isaBuffer);
src[i] = numIter;
dst[i] = 0xff;
dispatch[i]->SetArgs(&src[i], &dst[i]);
dispatch[i]->SetDim(1, 1, 1);
dispatch[i]->Submit(queues[i]);
}
for (i = 0; i < MAX_CP_QUEUES; ++i) {
dispatch[i]->Sync();
EXPECT_EQ_GPU(dst[i], src[i], gpuNode);
delete dispatch[i];
}
++numIter;
curTime = GetSystemTickCountInMicroSec();
timePassed = (curTime - initialTime) / 1000000;
} while (timePassed < TEST_TIME_SEC);
LOG() << "Total iterated : " << std::dec << numIter << std::endl;
for (i = 0; i < MAX_CP_QUEUES; ++i)
EXPECT_SUCCESS_GPU(queues[i].Destroy(), gpuNode);
}
TEST_F(KFDQMTest, MultipleCpQueuesStressDispatch) {
TEST_START(TESTPROFILE_RUNALL)
ASSERT_SUCCESS(KFDTestLaunch([this](int gpuNode) {
this->MultipleCpQueuesStressDispatch(gpuNode);
}));
TEST_END
}
void KFDQMTest::CpuWriteCoherence(int gpuNode) {
PM4Queue queue;
HsaMemoryBuffer destBuf(PAGE_SIZE, gpuNode);
ASSERT_SUCCESS_GPU(queue.Create(gpuNode), gpuNode);
HsaEvent *event;
ASSERT_SUCCESS_GPU(CreateQueueTypeEvent(false, false, gpuNode, &event), gpuNode);
/* The queue might be full and we fail to submit. There is always one word space unused in queue.
* So let rptr one step ahead then we continually submit packet.
*/
queue.PlaceAndSubmitPacket(PM4NopPacket());
queue.Wait4PacketConsumption();
EXPECT_EQ(1, queue.Rptr());
do {
queue.PlaceAndSubmitPacket(PM4NopPacket());
} while (queue.Wptr() != 0);
queue.Wait4PacketConsumption();
EXPECT_EQ_GPU(0, queue.Rptr(), gpuNode);
/* Now that the GPU has cached the PQ contents, we modify them in CPU cache and
* ensure that the GPU sees the updated value:
*/
queue.PlaceAndSubmitPacket(PM4WriteDataPacket(destBuf.As<unsigned int*>(), 0x42, 0x42));
queue.Wait4PacketConsumption(event);
WaitOnValue(destBuf.As<unsigned int*>(), 0x42);
hsaKmtDestroyEvent(event);
}
TEST_F(KFDQMTest, CpuWriteCoherence) {
TEST_START(TESTPROFILE_RUNALL);
ASSERT_SUCCESS(KFDTestLaunch([this](int gpuNode) {
this->CpuWriteCoherence(gpuNode);
}));
TEST_END
}
void KFDQMTest::CreateAqlCpQueue(int gpuNode) {
AqlQueue queue;
HsaMemoryBuffer pointers(PAGE_SIZE, gpuNode, /*zero*/true, /*local*/false, /*exec*/false, /*isScratch */false, /* isReadOnly */false, /* isUncached */false, /* NonPaged */g_baseTest->NeedNonPagedWptr(gpuNode));
ASSERT_SUCCESS_GPU(queue.Create(gpuNode, PAGE_SIZE, pointers.As<HSAuint64 *>()), gpuNode);
EXPECT_SUCCESS_GPU(queue.Destroy(), gpuNode);
}
TEST_F(KFDQMTest, CreateAqlCpQueue) {
TEST_START(TESTPROFILE_RUNALL)
ASSERT_SUCCESS(KFDTestLaunch([this](int gpuNode) {
this->CreateAqlCpQueue(gpuNode);
}));
TEST_END
}
void KFDQMTest::QueueLatency(int gpuNode) {
HSAuint32 m_FamilyId = GetFamilyIdFromNodeId(gpuNode);
PM4Queue queue;
const int queueSize = PAGE_SIZE * 2;
const int packetSize = PM4ReleaseMemoryPacket(m_FamilyId, 0, 0, 0, 0, 0).SizeInBytes();
/* We always leave one NOP(dword) empty after packet which is required by ring itself.
* We also place NOPs when queue wraparound to avoid crossing buffer end. See PlacePacket().
* So the worst case is that we need two packetSize space to place one packet.
* Like below, N=NOP,E=Empty,P=Packet.
* |E|E|E|E|E|E|E|rptr...wptr|E|E|E|E|E| ---> |P|P|P|P|P|P|E|rptr...wptr|N|N|N|N|N|
* So to respect that, we reserve packetSize space for these additional NOPs.
* Also we reserve the remainder of the division by packetSize explicitly.
* Reserve another packetSize for event-based wait which uses a releseMemory packet.
*/
const int reservedSpace = packetSize + queueSize % packetSize + packetSize;
const int slots = (queueSize - reservedSpace) / packetSize;
HSAint64 queue_latency_avg = 0, queue_latency_min, queue_latency_max, queue_latency_med;
HSAint64 overhead, workload;
HSAint64 *queue_latency_arr = reinterpret_cast<HSAint64*>(calloc(slots, sizeof(HSAint64)));
const int skip = 2;
const char *fs[skip] = {"1st", "2nd"};
HsaClockCounters *ts;
HSAuint64 *qts;
int i = 0;
ASSERT_NE_GPU((HSAuint64)queue_latency_arr, 0, gpuNode);
ASSERT_SUCCESS_GPU(queue.Create(gpuNode, queueSize), gpuNode);
LOG() << std::dec << "Queue Submit NanoSeconds (" << slots << " Packets)" << std::endl;
HsaMemoryBuffer buf(ALIGN_UP(slots * sizeof(HsaClockCounters), PAGE_SIZE), 0);
ts = buf.As<HsaClockCounters*>();
HsaMemoryBuffer qbuf(ALIGN_UP(slots * sizeof(HSAuint64), PAGE_SIZE), 0);
qts = qbuf.As<HSAuint64*>();
HsaEvent *event;
ASSERT_SUCCESS_GPU(CreateQueueTypeEvent(false, false, gpuNode, &event), gpuNode);
/* GpuCounter overhead*/
do {
hsaKmtGetClockCounters(gpuNode, &ts[i]);
} while (++i < slots);
overhead = ts[slots-1].GPUClockCounter - ts[0].GPUClockCounter;
overhead /= 2 * (slots - 1);
/* Submit packets serially*/
i = 0;
do {
queue.PlacePacket(PM4ReleaseMemoryPacket(m_FamilyId, true,
(HSAuint64)&qts[i],
0,
true,
1));
hsaKmtGetClockCounters(gpuNode, &ts[i]);
queue.SubmitPacket();
queue.Wait4PacketConsumption(event);
} while (++i < slots);
/* Calculate timing which includes workload and overhead*/
i = 0;
do {
HSAint64 queue_latency = qts[i] - ts[i].GPUClockCounter;
EXPECT_GE_GPU(queue_latency, 0, gpuNode);
queue_latency_arr[i] = queue_latency;
if (i >= skip)
queue_latency_avg += queue_latency;
} while (++i < slots);
/* Calculate avg from packet[skip, slots-1] */
queue_latency_avg /= (slots - skip);
/* Workload of queue packet itself */
i = 0;
do {
queue.PlacePacket(PM4ReleaseMemoryPacket(m_FamilyId, true,
(HSAuint64)&qts[i],
0,
true,
1));
} while (++i < slots);
queue.SubmitPacket();
queue.Wait4PacketConsumption(event);
hsaKmtDestroyEvent(event);
/* qts[i] records the timestamp of the end of packet[i] which is
* approximate that of the beginging of packet[i+1].
* The workload total is [0, skip], [skip+1, slots-1].
* And We ignore [0, skip], that means we ignore (skip+1) packets.
*/
workload = qts[slots - 1] - qts[skip];
workload /= (slots - 1 - skip);
EXPECT_GE_GPU(workload, 0, gpuNode);
i = 0;
do {
/* The queue_latency is not that correct as the workload and overhead are average*/
queue_latency_arr[i] -= workload + overhead;
/* The First submit takes an HSAint64 time*/
if (i < skip)
LOG() << "Queue Latency " << fs[i] << ": \t" << CounterToNanoSec(queue_latency_arr[i]) << std::endl;
} while (++i < slots);
std::sort(queue_latency_arr + skip, queue_latency_arr + slots);
queue_latency_min = queue_latency_arr[skip];
queue_latency_med = queue_latency_arr[(slots+skip)/2];
queue_latency_max = queue_latency_arr[slots-1];
LOG() << "Queue Latency Avg: \t" << CounterToNanoSec(queue_latency_avg) << std::endl;
LOG() << "Queue Latency Min: \t" << CounterToNanoSec(queue_latency_min) << std::endl;
LOG() << "Queue Latency Median: \t" << CounterToNanoSec(queue_latency_med) << std::endl;
LOG() << "Queue Latency Max: \t" << CounterToNanoSec(queue_latency_max) << std::endl;
LOG() << "Queue Packet Workload: \t" << CounterToNanoSec(workload) << std::endl;
LOG() << "Get GpuCounter Overhead: \t" << CounterToNanoSec(overhead) << std::endl;
RECORD(CounterToNanoSec(queue_latency_avg)) << "Queue-Latency-Avg";
RECORD(CounterToNanoSec(queue_latency_min)) << "Queue-Latency-Min";
RECORD(CounterToNanoSec(queue_latency_med)) << "Queue-Latency-Med";
RECORD(CounterToNanoSec(queue_latency_max)) << "Queue-Latency-Max";
RECORD(CounterToNanoSec(workload)) << "Queue-Packet-Workload";
RECORD(CounterToNanoSec(overhead)) << "GpuCounter-Overhead";
}
TEST_F(KFDQMTest, QueueLatency) {
TEST_START(TESTPROFILE_RUNALL);
ASSERT_SUCCESS(KFDTestLaunch([this](int gpuNode) {
this->QueueLatency(gpuNode);
}));
TEST_END
}
void KFDQMTest::CpQueueWraparound(int gpuNode) {
PM4Queue queue;
HsaMemoryBuffer destBuf(PAGE_SIZE, gpuNode);
ASSERT_SUCCESS_GPU(queue.Create(gpuNode), gpuNode);
HsaEvent *event;
ASSERT_SUCCESS_GPU(CreateQueueTypeEvent(false, false, gpuNode, &event), gpuNode);
for (unsigned int pktIdx = 0; pktIdx <= PAGE_SIZE/sizeof(PM4WRITE_DATA_CI); ++pktIdx) {
queue.PlaceAndSubmitPacket(PM4WriteDataPacket(destBuf.As<unsigned int*>(), pktIdx, pktIdx));
queue.Wait4PacketConsumption(event);
WaitOnValue(destBuf.As<unsigned int*>(), pktIdx);
}
for (unsigned int pktIdx = 0; pktIdx <= PAGE_SIZE/sizeof(PM4WRITE_DATA_CI); ++pktIdx) {
queue.PlaceAndSubmitPacket(PM4WriteDataPacket(destBuf.As<unsigned int*>(), pktIdx, pktIdx));
queue.Wait4PacketConsumption(event);
WaitOnValue(destBuf.As<unsigned int*>(), pktIdx);
}
hsaKmtDestroyEvent(event);
EXPECT_SUCCESS_GPU(queue.Destroy(), gpuNode);
}
TEST_F(KFDQMTest, CpQueueWraparound) {
TEST_START(TESTPROFILE_RUNALL);
ASSERT_SUCCESS(KFDTestLaunch([this](int gpuNode) {
this->CpQueueWraparound(gpuNode);
}));
TEST_END
}
void KFDQMTest::SdmaQueueWraparound(int gpuNode) {
int bufSize = PAGE_SIZE;
SDMAQueue queue;
HsaMemoryBuffer destBuf(bufSize << 1, gpuNode, false);
HsaMemoryBuffer srcBuf(bufSize, gpuNode, false);
ASSERT_SUCCESS_GPU(queue.Create(gpuNode), gpuNode);
for (unsigned int pktIdx = 0; pktIdx <= queue.Size()/sizeof(SDMA_PKT_COPY_LINEAR); ++pktIdx) {
destBuf.Fill(0x0);
srcBuf.Fill(pktIdx);
queue.PlaceAndSubmitPacket(
SDMACopyDataPacket(queue.GetFamilyId(), destBuf.As<unsigned int*>(), srcBuf.As<unsigned int*>(), bufSize));
queue.PlaceAndSubmitPacket(
SDMAWriteDataPacket(queue.GetFamilyId(), destBuf.As<unsigned int*>() + bufSize/4, 0x02020202));
queue.Wait4PacketConsumption();
EXPECT_TRUE_GPU(WaitOnValue(destBuf.As<unsigned int*>() + bufSize/4, 0x02020202), gpuNode);
EXPECT_SUCCESS_GPU(memcmp(
destBuf.As<unsigned int*>(), srcBuf.As<unsigned int*>(), bufSize), gpuNode);
}
for (unsigned int pktIdx = 0; pktIdx <= queue.Size()/sizeof(SDMA_PKT_WRITE_UNTILED); ++pktIdx) {
queue.PlaceAndSubmitPacket(SDMAWriteDataPacket(queue.GetFamilyId(), destBuf.As<unsigned int*>(), pktIdx));
queue.Wait4PacketConsumption();
WaitOnValue(destBuf.As<unsigned int*>(), pktIdx);
}
EXPECT_SUCCESS_GPU(queue.Destroy(), gpuNode);
}
TEST_F(KFDQMTest, SdmaQueueWraparound) {
TEST_START(TESTPROFILE_RUNALL);
ASSERT_SUCCESS(KFDTestLaunch([this](int gpuNode) {
this->SdmaQueueWraparound(gpuNode);
}));
TEST_END
}
struct AtomicIncThreadParams {
HSAint64* pDest;
volatile unsigned int count;
volatile bool loop;
};
unsigned int AtomicIncThread(void* pCtx) {
AtomicIncThreadParams* pArgs = reinterpret_cast<AtomicIncThreadParams*>(pCtx);
while (pArgs->loop) {
AtomicInc(pArgs->pDest);
++pArgs->count;
}
LOG() << "CPU atomic increments finished" << std::endl;
return 0;
}
void KFDQMTest::Atomics(int gpuNode) {
Assembler* m_pAsm;
m_pAsm = GetAssemblerFromNodeId(gpuNode);
ASSERT_NOTNULL_GPU(m_pAsm, gpuNode);
if (!hasPciAtomicsSupport(gpuNode)) {
LOG() << "Skipping test: Node doesn't support Atomics." << std::endl;
return;
}
HsaMemoryBuffer isaBuf(PAGE_SIZE, gpuNode, true/*zero*/, false/*local*/, true/*exec*/);
HsaMemoryBuffer destBuf(PAGE_SIZE, gpuNode);
PM4Queue queue;
ASSERT_SUCCESS_GPU(m_pAsm->RunAssembleBuf(AtomicIncIsa, isaBuf.As<char*>()), gpuNode);
Dispatch dispatch(isaBuf);
dispatch.SetArgs(destBuf.As<void*>(), NULL);
dispatch.SetDim(1024, 1, 1);
hsaKmtSetMemoryPolicy(gpuNode, HSA_CACHING_CACHED, HSA_CACHING_CACHED, NULL, 0);
ASSERT_SUCCESS_GPU(queue.Create(gpuNode), gpuNode);
AtomicIncThreadParams params;
params.pDest = destBuf.As<HSAint64*>();
params.loop = true;
params.count = 0;
uint64_t threadId;
ASSERT_EQ_GPU(true, StartThread(&AtomicIncThread, &params, threadId), gpuNode);
LOG() << "Waiting for CPU to atomic increment 1000 times" << std::endl;
while (params.count < 1000)
{}
LOG() << "Submitting the GPU atomic increment shader" << std::endl;
dispatch.Submit(queue);
dispatch.Sync();
params.loop = false;
WaitForThread(threadId);
EXPECT_EQ_GPU(destBuf.As<unsigned int*>()[0], 1024 + params.count, gpuNode);
LOG() << "GPU increments: 1024, CPU increments: " << std::dec
<< params.count << std::endl;
queue.Destroy();
}
TEST_F(KFDQMTest, Atomics) {
TEST_START(TESTPROFILE_RUNALL);
ASSERT_SUCCESS(KFDTestLaunch([this](int gpuNode) {
this->Atomics(gpuNode);
}));
TEST_END
}
TEST_F(KFDQMTest, mGPUShareBO) {
TEST_START(TESTPROFILE_RUNALL);
unsigned int src_node = 2;
unsigned int dst_node = 1;
if (g_TestDstNodeId != -1 && g_TestNodeId != -1) {
src_node = g_TestNodeId;
dst_node = g_TestDstNodeId;
}
HsaMemoryBuffer shared_addr(PAGE_SIZE, dst_node, true, false, false, false);
HsaMemoryBuffer srcNodeMem(PAGE_SIZE, src_node);
HsaMemoryBuffer dstNodeMem(PAGE_SIZE, dst_node);
/* Handle ISA to write to local memory BO */
HsaMemoryBuffer isaBufferSrc(PAGE_SIZE, src_node, true/*zero*/, false/*local*/, true/*exec*/);
HsaMemoryBuffer isaBufferDst(PAGE_SIZE, dst_node, true/*zero*/, false/*local*/, true/*exec*/);
srcNodeMem.Fill(0x05050505);
ASSERT_SUCCESS(m_pAsm->RunAssemble(CopyDwordIsa));
m_pAsm->CopyInstrStream(isaBufferSrc.As<char*>());
SyncDispatch(isaBufferSrc, srcNodeMem.As<void*>(), shared_addr.As<void *>(), src_node);
m_pAsm->CopyInstrStream(isaBufferDst.As<char*>());
SyncDispatch(isaBufferDst, shared_addr.As<void *>(), dstNodeMem.As<void*>(), dst_node);
EXPECT_EQ(dstNodeMem.As<unsigned int*>()[0], 0x05050505);
EXPECT_SUCCESS(shared_addr.UnmapMemToNodes(&dst_node, 1));
TEST_END
}
static void
sdma_copy(HSAuint32 node, void *src, void *const dst[], int n, HSAuint64 size) {
SDMAQueue sdmaQueue;
HsaEvent *event;
ASSERT_SUCCESS(CreateQueueTypeEvent(false, false, node, &event));
ASSERT_SUCCESS(sdmaQueue.Create(node));
sdmaQueue.PlaceAndSubmitPacket(SDMACopyDataPacket(sdmaQueue.GetFamilyId(), dst, src, n, size));
sdmaQueue.Wait4PacketConsumption(event);
EXPECT_SUCCESS(sdmaQueue.Destroy());
hsaKmtDestroyEvent(event);
}
static void
sdma_fill(HSAint32 node, void *dst, unsigned int data, HSAuint64 size) {
SDMAQueue sdmaQueue;
HsaEvent *event;
ASSERT_SUCCESS(CreateQueueTypeEvent(false, false, node, &event));
ASSERT_SUCCESS(sdmaQueue.Create(node));
sdmaQueue.PlaceAndSubmitPacket(SDMAFillDataPacket(sdmaQueue.GetFamilyId(), dst, data, size));
sdmaQueue.Wait4PacketConsumption(event);
EXPECT_SUCCESS(sdmaQueue.Destroy());
hsaKmtDestroyEvent(event);
}
TEST_F(KFDQMTest, P2PTest) {
TEST_START(TESTPROFILE_RUNALL);
if (!hsakmt_is_dgpu()) {
LOG() << "Skipping test: Two GPUs are required, but no dGPUs are present." << std::endl;
return;
}
const std::vector<int> gpuNodes = m_NodeInfo.GetNodesWithGPU();
if (gpuNodes.size() < 2) {
LOG() << "Skipping test: At least two GPUs are required." << std::endl;
return;
}
std::vector<int> nodes;
/* This test simulates RT team's P2P part in IPCtest:
*
* +------------------------------------------------+
* | gpu1 gpu2 gpuX |
* |gpu1 mem ----> gpu2 mem ----> gpuX mem |
* | \ \ \ mGPUShareBO |
* | \ \ \ |
* | system buffer system buffer system buffer|
* +------------------------------------------------+
*
* Copy data from current GPU memory to next GPU memory and system memory
* Using current GPU, aka p2p push.
* Verify the system buffer has the expected content after each push.
*/
/* Users can use "--node=gpu1 --dst_node=gpu2" to specify devices */
if (g_TestDstNodeId != -1 && g_TestNodeId != -1) {
nodes.push_back(g_TestNodeId);
nodes.push_back(g_TestDstNodeId);
if (!m_NodeInfo.IsPeerAccessibleByNode(g_TestNodeId, g_TestDstNodeId)) {
LOG() << "Skipping test: Dst GPU specified is not peer-accessible." << std::endl;
return;
}
if (nodes[0] == nodes[1]) {
LOG() << "Skipping test: Different GPUs must be specified (2 GPUs required)." << std::endl;
return;
}
} else {
nodes = m_NodeInfo.GetNodesWithGPU();
if (nodes.size() < 2) {
LOG() << "Skipping test: Test requires at least one large bar GPU." << std::endl;
LOG() << " or two GPUs are XGMI connected." << std::endl;
return;
}
}
HSAuint32 *sysBuf;
HSAuint32 size = 16ULL<<20; // bigger than 16MB to test non-contiguous memory
HsaMemFlags memFlags = {0};
HsaMemMapFlags mapFlags = {0};
memFlags.ui32.PageSize = HSA_PAGE_SIZE_4KB;
memFlags.ui32.HostAccess = 0;
memFlags.ui32.NonPaged = 1;
memFlags.ui32.NoNUMABind = 1;
unsigned int end = size / sizeof(HSAuint32) - 1;
/* 1. Allocate a system buffer and allow the access to GPUs */
EXPECT_SUCCESS(hsaKmtAllocMemory(0, size, m_MemoryFlags,
reinterpret_cast<void **>(&sysBuf)));
EXPECT_SUCCESS(hsaKmtMapMemoryToGPUNodes(sysBuf, size, NULL,
mapFlags, nodes.size(), (HSAuint32 *)&nodes[0]));
#define MAGIC_NUM 0xdeadbeaf
/* First GPU fills mem with MAGIC_NUM */
void *src, *dst;
HSAuint32 cur = nodes[0], next;
ASSERT_SUCCESS(hsaKmtAllocMemory(cur, size, memFlags, reinterpret_cast<void**>(&src)));
ASSERT_SUCCESS(hsaKmtMapMemoryToGPU(src, size, NULL));
sdma_fill(cur, src, MAGIC_NUM, size);
for (unsigned i = 1; i <= nodes.size(); i++) {
int n;
memset(sysBuf, 0, size);
/* Last GPU just copy mem to sysBuf*/
if (i == nodes.size()) {
n = 1;
next = 0;/*system memory node*/
dst = 0;
} else {
n = 2;
next = nodes[i];
/* check if cur access next node */
if (!m_NodeInfo.IsPeerAccessibleByNode(next, cur))
continue;
ASSERT_SUCCESS(hsaKmtAllocMemory(next, size, memFlags, reinterpret_cast<void**>(&dst)));
ASSERT_SUCCESS(hsaKmtMapMemoryToGPU(dst, size, NULL));
}
LOG() << "Test " << cur << " -> " << next << std::endl;
/* Copy to sysBuf and next GPU*/
void *dst_array[] = {sysBuf, dst};
sdma_copy(cur, src, dst_array, n, size);
/* Verify the data*/
EXPECT_EQ(sysBuf[0], MAGIC_NUM);
EXPECT_EQ(sysBuf[end], MAGIC_NUM);
LOG() << "PASS " << cur << " -> " << next << std::endl;
EXPECT_SUCCESS(hsaKmtUnmapMemoryToGPU(src));
EXPECT_SUCCESS(hsaKmtFreeMemory(src, size));
cur = next;
src = dst;
}
EXPECT_SUCCESS(hsaKmtUnmapMemoryToGPU(sysBuf));
EXPECT_SUCCESS(hsaKmtFreeMemory(sysBuf, size));
TEST_END
}
void KFDQMTest::PM4EventInterrupt(int gpuNode) {
HSAuint32 m_FamilyId = GetFamilyIdFromNodeId(gpuNode);
const HSAuint64 bufSize = PAGE_SIZE;
const int packetCount = bufSize / sizeof(unsigned int);
const int totalPacketSize = packetCount * PM4WriteDataPacket(0, 0).SizeInBytes() +
PM4ReleaseMemoryPacket(m_FamilyId, 0, 0, 0).SizeInBytes();
const int queueSize = RoundToPowerOf2(totalPacketSize);
/* Reduce number of iteration if running with emulator. */
const int numIter = (g_IsEmuMode ? 32 : 1024);
/* 4 PM4 queues will be running at same time.*/
const int numPM4Queue = 4;
HsaEvent *event[numPM4Queue];
PM4Queue queue[numPM4Queue];
HsaMemoryBuffer *destBuf[numPM4Queue];
unsigned int *buf[numPM4Queue];
for (int i = 0; i < numPM4Queue; i++) {
destBuf[i] = new HsaMemoryBuffer(bufSize, gpuNode, true, false); // System memory
buf[i] = destBuf[i]->As<unsigned int *>();
}
/* A simple loop here to give more pressure.*/
for (int test_count = 0; test_count < numIter; test_count++) {
for (int i = 0; i < numPM4Queue; i++) {
ASSERT_SUCCESS_GPU(queue[i].Create(gpuNode, queueSize), gpuNode);
ASSERT_SUCCESS_GPU(CreateQueueTypeEvent(false, false, gpuNode, &event[i]), gpuNode);
/* Let CP have some workload first.*/
for(int index = 0; index < packetCount; index++)
queue[i].PlacePacket(PM4WriteDataPacket(buf[i] + index, 0xdeadbeaf));
/* releaseMemory packet makes sure all previous written data is visible.*/
queue[i].PlacePacket(PM4ReleaseMemoryPacket(m_FamilyId, 0,
reinterpret_cast<HSAuint64>(event[i]->EventData.HWData2),
event[i]->EventId,
true));
}
for (int i = 0; i < numPM4Queue; i++)
queue[i].SubmitPacket();
for (int i = 0; i < numPM4Queue; i++) {
EXPECT_SUCCESS_GPU(hsaKmtWaitOnEvent(event[i], g_TestTimeOut), gpuNode);
EXPECT_EQ_GPU(buf[i][0], 0xdeadbeaf, gpuNode);
EXPECT_EQ_GPU(buf[i][packetCount - 1], 0xdeadbeaf, gpuNode);
memset(buf[i], 0, bufSize);
}
for (int i = 0; i < numPM4Queue; i++) {
EXPECT_SUCCESS_GPU(queue[i].Destroy(), gpuNode);
EXPECT_SUCCESS_GPU(hsaKmtDestroyEvent(event[i]), gpuNode);
}
}
for (int i = 0; i < numPM4Queue; i++)
delete destBuf[i];
}
TEST_F(KFDQMTest, PM4EventInterrupt) {
TEST_START(TESTPROFILE_RUNALL)
ASSERT_SUCCESS(KFDTestLaunch([this](int gpuNode) {
this->PM4EventInterrupt(gpuNode);
}));
TEST_END
}
#include "KFDTestUtilQueue.hpp"
void KFDQMTest::SdmaEventInterrupt(int gpuNode) {
const HSAuint64 bufSize = 4 << 20;
HsaMemoryBuffer srcBuf(bufSize, 0); // System memory.
HSAuint64 *src = srcBuf.As<HSAuint64*>();
TimeStamp *tsbuf = srcBuf.As<TimeStamp*>();
tsbuf = reinterpret_cast<TimeStamp *>ALIGN_UP(tsbuf, sizeof(TimeStamp));
/* Have 3 queues created for test.*/
const int numSDMAQueue = 3;
HsaEvent *event[numSDMAQueue];
SDMAQueue queue[numSDMAQueue];
HsaMemoryBuffer *destBuf[numSDMAQueue];
HSAuint64 *dst[numSDMAQueue];
for (int i = 0; i < numSDMAQueue; i++) {
destBuf[i] = new HsaMemoryBuffer(bufSize, gpuNode, true, false); // System memory
dst[i] = destBuf[i]->As<HSAuint64*>();
}
/* Test 1 queue, 2 queues, 3 queues running at same time one by one.*/
for (int testSDMAQueue = 1; testSDMAQueue <= numSDMAQueue; testSDMAQueue++)
/* A simple loop here to give more pressure.*/
for (int test_count = 0; test_count < 2048; test_count++) {
for (int i = 0; i < testSDMAQueue; i++) {
TimeStamp *ts = tsbuf + i * 32;
ASSERT_SUCCESS_GPU(queue[i].Create(gpuNode), gpuNode);
/* FIXME
* We create event every time along with queue.
* However that will significantly enhance the failure of sdma event timeout.
*/
ASSERT_SUCCESS_GPU(CreateQueueTypeEvent(false, false, gpuNode, &event[i]), gpuNode);
/* Get the timestamp directly. The first member of HsaClockCounters and TimeStamp is GPU clock counter.*/
hsaKmtGetClockCounters(gpuNode, reinterpret_cast<HsaClockCounters*>(&ts[0]));
/* Let sDMA have some workload first.*/
queue[i].PlacePacket(SDMATimePacket(&ts[1]));
queue[i].PlacePacket(
SDMACopyDataPacket(queue[i].GetFamilyId(), dst[i], src, bufSize));
queue[i].PlacePacket(SDMATimePacket(&ts[2]));
queue[i].PlacePacket(
SDMAFencePacket(queue[i].GetFamilyId(),
reinterpret_cast<void*>(event[i]->EventData.HWData2), event[i]->EventId));
queue[i].PlacePacket(SDMATimePacket(&ts[3]));
queue[i].PlacePacket(SDMATrapPacket(event[i]->EventId));
queue[i].PlacePacket(SDMATimePacket(&ts[4]));
/* Will verify the value of srcBuf and destBuf later. Give it a different value each time.*/
src[0] = ts[0].timestamp;
}
for (int i = 0; i < testSDMAQueue; i++)
queue[i].SubmitPacket();
for (int i = 0; i < testSDMAQueue; i++) {
TimeStamp *ts = tsbuf + i * 32;
HSAKMT_STATUS ret = hsaKmtWaitOnEvent(event[i], g_TestTimeOut);
if (dst[i][0] != src[0])
WARN() << "SDMACopyData FAIL! " << std::dec
<< dst[i][0] << " VS " << src[0] << std::endl;
if (ret == HSAKMT_STATUS_SUCCESS) {
for (int i = 1; i <= 4; i++)
/* Is queue latency too big? The workload is really small.*/
if (CounterToNanoSec(ts[i].timestamp - ts[i - 1].timestamp) > 1000000000)
WARN() << "SDMA queue latency is bigger than 1s!" << std::endl;
} else {
WARN() << "Event On Queue " << testSDMAQueue << ":" << i
<< " Timeout, try to resubmit packets!" << std::endl;
queue[i].SubmitPacket();
if (hsaKmtWaitOnEvent(event[i], g_TestTimeOut) == HSAKMT_STATUS_SUCCESS)
WARN() << "The timeout event is signaled!" << std::endl;
else
WARN() << "The timeout event is lost after resubmit!" << std::endl;
LOG() << "Time Consumption (ns)" << std::endl;
for (int i = 1; i <= 4; i++)
LOG() << std::dec << i << ": "
<< CounterToNanoSec(ts[i].timestamp - ts[i - 1].timestamp) << std::endl;
}
EXPECT_SUCCESS_GPU(ret, gpuNode);
}
for (int i = 0; i < testSDMAQueue; i++) {
EXPECT_SUCCESS_GPU(queue[i].Destroy(), gpuNode);
EXPECT_SUCCESS_GPU(hsaKmtDestroyEvent(event[i]), gpuNode);
}
}
for (int i = 0; i < numSDMAQueue; i++)
delete destBuf[i];
}
TEST_F(KFDQMTest, SdmaEventInterrupt) {
TEST_START(TESTPROFILE_RUNALL)
ASSERT_SUCCESS(KFDTestLaunch([this](int gpuNode) {
this->SdmaEventInterrupt(gpuNode);
}));
TEST_END
}
#define DOORBELL_WRITE_USE_SDMA
void KFDQMTest::GPUDoorbellWrite(int gpuNode) {
HSAuint32 m_FamilyId = GetFamilyIdFromNodeId(gpuNode);
HsaMemoryBuffer destBuf(PAGE_SIZE, 0, true);
PM4Queue pm4Queue;
#ifdef DOORBELL_WRITE_USE_SDMA
SDMAQueue otherQueue;
#else
PM4Queue otherQueue;
#endif
ASSERT_SUCCESS_GPU(pm4Queue.Create(gpuNode), gpuNode);
ASSERT_SUCCESS_GPU(otherQueue.Create(gpuNode), gpuNode);
/* Place PM4 packet in the queue, but don't submit it */
pm4Queue.PlacePacket(PM4WriteDataPacket(destBuf.As<unsigned int*>(), 0x12345678, 0x87654321));
HsaQueueResource *qRes = pm4Queue.GetResource();
if (m_FamilyId < FAMILY_AI) {
unsigned int pendingWptr = pm4Queue.GetPendingWptr();
#ifdef DOORBELL_WRITE_USE_SDMA
/* Write the wptr and doorbell update using the GPU's SDMA
* engine. This should submit the PM4 packet on the first
* queue.
*/
otherQueue.PlacePacket(SDMAWriteDataPacket(otherQueue.GetFamilyId(), qRes->Queue_write_ptr,
pendingWptr));
otherQueue.PlacePacket(SDMAWriteDataPacket(otherQueue.GetFamilyId(), qRes->Queue_DoorBell,
pendingWptr));
#else
/* Write the wptr and doorbell update using WRITE_DATA packets
* on a second PM4 queue. This should submit the PM4 packet on
* the first queue.
*/
otherQueue.PlacePacket(
PM4ReleaseMemoryPacket(m_FamilyId, true, (HSAuint64)qRes->Queue_write_ptr,
pendingWptr, false));
otherQueue.PlacePacket(
PM4ReleaseMemoryPacket(m_FamilyId, true, (HSAuint64)qRes->Queue_DoorBell,
pendingWptr, false));
#endif
otherQueue.SubmitPacket();
} else {
HSAuint64 pendingWptr64 = pm4Queue.GetPendingWptr64();
#ifdef DOORBELL_WRITE_USE_SDMA
/* Write the wptr and doorbell update using the GPU's SDMA
* engine. This should submit the PM4 packet on the first
* queue.
*/
otherQueue.PlacePacket(SDMAWriteDataPacket(otherQueue.GetFamilyId(), qRes->Queue_write_ptr,
2, &pendingWptr64));
otherQueue.PlacePacket(SDMAWriteDataPacket(otherQueue.GetFamilyId(), qRes->Queue_DoorBell,
2, &pendingWptr64));
#else
/* Write the 64-bit wptr and doorbell update using RELEASE_MEM
* packets without IRQs on a second PM4 queue. RELEASE_MEM
* should perform one atomic 64-bit access. This should submit
* the PM4 packet on the first queue.
*/
otherQueue.PlacePacket(
PM4ReleaseMemoryPacket(m_FamilyId, true, (HSAuint64)qRes->Queue_write_ptr,
pendingWptr64, true));
otherQueue.PlacePacket(
PM4ReleaseMemoryPacket(m_FamilyId, true, (HSAuint64)qRes->Queue_DoorBell,
pendingWptr64, true));
#endif
otherQueue.SubmitPacket();
}
/* Check that the PM4 packet has been executed */
EXPECT_TRUE_GPU(WaitOnValue(destBuf.As<unsigned int *>(), 0x12345678), gpuNode);
EXPECT_TRUE_GPU(WaitOnValue(destBuf.As<unsigned int *>()+1, 0x87654321), gpuNode);
EXPECT_SUCCESS_GPU(pm4Queue.Destroy(), gpuNode);
EXPECT_SUCCESS_GPU(otherQueue.Destroy(), gpuNode);
}
TEST_F(KFDQMTest, GPUDoorbellWrite) {
TEST_START(TESTPROFILE_RUNALL)
ASSERT_SUCCESS(KFDTestLaunch([this](int gpuNode) {
this->GPUDoorbellWrite(gpuNode);
}));
TEST_END
}
TEST_F(KFDQMTest, UserQueueBufValidation) {
TEST_START(TESTPROFILE_RUNALL)
int defaultGPUNode = m_NodeInfo.HsaDefaultGPUNode();
ASSERT_GE(defaultGPUNode, 0) << "failed to get default GPU Node";
HsaQueueResource QueueResources;
HsaMemoryBuffer *QueueBuf;
HSAKMT_STATUS status;
memset(&QueueResources, 0, sizeof(QueueResources));
// System memory mapping on GPU
QueueBuf = new HsaMemoryBuffer(PAGE_SIZE, defaultGPUNode);
EXPECT_SUCCESS(hsaKmtCreateQueue(defaultGPUNode,
HSA_QUEUE_COMPUTE,
100,
HSA_QUEUE_PRIORITY_NORMAL,
QueueBuf->As<unsigned int*>(),
PAGE_SIZE,
NULL,
&QueueResources));
EXPECT_SUCCESS(hsaKmtDestroyQueue(QueueResources.QueueId));
// CP Queue creation should fail using wrong ring buffer size
EXPECT_SUCCESS(!hsaKmtCreateQueue(defaultGPUNode,
HSA_QUEUE_COMPUTE,
100,
HSA_QUEUE_PRIORITY_NORMAL,
QueueBuf->As<unsigned int*>(),
PAGE_SIZE * 2,
NULL,
&QueueResources));
// SDMA queue create should fail using wrong ring buffer size
EXPECT_SUCCESS(!hsaKmtCreateQueue(defaultGPUNode,
HSA_QUEUE_SDMA,
100,
HSA_QUEUE_PRIORITY_NORMAL,
QueueBuf->As<unsigned int*>(),
PAGE_SIZE * 2,
NULL,
&QueueResources));
// CP queue create should fail using NULL ring buffer
EXPECT_SUCCESS(!hsaKmtCreateQueue(defaultGPUNode,
HSA_QUEUE_COMPUTE,
100,
HSA_QUEUE_PRIORITY_NORMAL,
NULL,
PAGE_SIZE,
NULL,
&QueueResources));
// SDMA queue create should fail using NULL ring buffer
EXPECT_SUCCESS(!hsaKmtCreateQueue(defaultGPUNode,
HSA_QUEUE_SDMA,
100,
HSA_QUEUE_PRIORITY_NORMAL,
NULL,
PAGE_SIZE,
NULL,
&QueueResources));
EXPECT_SUCCESS(hsaKmtUnmapMemoryToGPU(QueueBuf->As<unsigned int*>()));
EXPECT_SUCCESS(hsaKmtFreeMemory(QueueBuf->As<unsigned int*>(), PAGE_SIZE));
//
// This following negative test will evict user queues, must execute in child process,
// because parent process is allowed to create queue to run the remaining tests.
//
pid_t childPid = fork();
if (childPid == 0) { /* Child process */
void *cwsr_addr;
int exit_code = 1;
TearDown();
SetUp();
// System memory mapping on GPU
QueueBuf = new HsaMemoryBuffer(PAGE_SIZE, defaultGPUNode);
memset(&QueueResources, 0, sizeof(QueueResources));
status = hsaKmtCreateQueue(defaultGPUNode,
HSA_QUEUE_COMPUTE,
100,
HSA_QUEUE_PRIORITY_NORMAL,
QueueBuf->As<unsigned int*>(),
PAGE_SIZE,
NULL,
&QueueResources);
if (status != HSAKMT_STATUS_SUCCESS) {
LOG() << "create queue failed." << std::endl;
goto free_exit;
}
// Update queue percentage 0 to set queue inactive in order to get queue info CWSR area
status = hsaKmtUpdateQueue(QueueResources.QueueId, 0, HSA_QUEUE_PRIORITY_NORMAL,
QueueBuf->As<unsigned int*>(), PAGE_SIZE, NULL);
if (status != HSAKMT_STATUS_SUCCESS) {
LOG() << "update queue failed." << std::endl;
goto err_exit;
}
HsaQueueInfo QueueInfo;
status = hsaKmtGetQueueInfo(QueueResources.QueueId, &QueueInfo);
if (status != HSAKMT_STATUS_SUCCESS) {
LOG() << "get queue info failed." << std::endl;
goto err_exit;
}
// unmap CWSR buffer will evict queue before queue is destroyed
cwsr_addr = QueueInfo.UserContextSaveArea;
munmap(cwsr_addr, PAGE_SIZE);
// unmap and free queue ring buffer should fail before the queue is destroyed
status = hsaKmtFreeMemory(QueueBuf->As<unsigned int*>(), PAGE_SIZE);
if (status == HSAKMT_STATUS_SUCCESS) {
LOG() << "free queue buf should fail." << std::endl;
goto err_exit;
}
status = hsaKmtUnmapMemoryToGPU(QueueBuf->As<unsigned int*>());
if (status == HSAKMT_STATUS_SUCCESS) {
LOG() << "unmap queue buf should fail." << std::endl;
goto err_exit;
}
exit_code = 0;
err_exit:
status = hsaKmtDestroyQueue(QueueResources.QueueId);
if (status != HSAKMT_STATUS_SUCCESS) {
LOG() << "destroy queue failed." << std::endl;
exit_code = 1;
}
free_exit:
status = hsaKmtUnmapMemoryToGPU(QueueBuf->As<unsigned int*>());
if (status != HSAKMT_STATUS_SUCCESS) {
LOG() << "unmap queue buf failed." << std::endl;
exit_code = 1;
}
status = hsaKmtFreeMemory(QueueBuf->As<unsigned int*>(), PAGE_SIZE);
if (status != HSAKMT_STATUS_SUCCESS) {
LOG() << "free queue buf failed." << std::endl;
exit_code = 1;
}
exit(exit_code);
} else {
int childStatus;
waitpid(childPid, &childStatus, 0);
EXPECT_EQ(true, WIFEXITED(childStatus));
EXPECT_EQ(0, WEXITSTATUS(childStatus));
}
TEST_END
}
Ссылка в новой задаче Показать git blame Копировать постоянную ссылку