KFDTest: remove the usage global g_TestGPUFamilyId

Adjust the KFDTest for multi-gou support Change-Id: Ib3491e3f645d35fdba6ab702d65fcc86f48d3958 Signed-off-by: shaoyunl <shaoyun.liu@amd.com> [ROCm/ROCR-Runtime commit: b4e834ab61]
2019-07-16 16:46:39 -04:00
@@ -23,6 +23,11 @@

 #include "BasePacket.hpp"
 #include "KFDTestUtil.hpp"
+#include "KFDBaseComponentTest.hpp"
+
+BasePacket::BasePacket(void) {
+    m_FamilyId = g_baseTest->GetFamilyIdFromDefaultNode();
+}

 void BasePacket::Dump() const {
    unsigned int size = SizeInDWords();
@@ -37,7 +37,7 @@ enum PACKETTYPE {
 // @class BasePacket
 class BasePacket {
 public:
-    BasePacket(void) {}
+    BasePacket(void);
    virtual ~BasePacket(void) {}

    // @returns Packet type
@@ -52,6 +52,7 @@ class BasePacket {
    void Dump() const;

 protected:
+     unsigned int m_FamilyId;
 };

 #endif  // __KFD_BASE_PACKET__H__
@@ -26,6 +26,7 @@
 #include "PM4Queue.hpp"
 #include "AqlQueue.hpp"
 #include "hsakmt.h"
+#include "KFDBaseComponentTest.hpp"

 BaseQueue::BaseQueue()
    :m_QueueBuf(NULL),
@@ -81,6 +82,7 @@ HSAKMT_STATUS BaseQueue::Create(unsigned int NodeId, unsigned int size, HSAuint6
    m_pendingWptr = 0;
    m_pendingWptr64 = 0;
    m_Node = NodeId;
+    m_FamilyId = g_baseTest->GetFamilyIdFromNodeId(NodeId);
    return status;
 }

@@ -87,6 +87,7 @@ class BaseQueue {
    HSAuint64 GetPendingWptr64() { return m_pendingWptr64; }
    virtual _HSA_QUEUE_TYPE GetQueueType() = 0;
    unsigned int GetNodeId() { return m_Node; }
+    unsigned int GetFamilyId() { return m_FamilyId; }

 protected:
    static const unsigned int CMD_NOP_TYPE_2        = 0x80000000;
@@ -98,6 +99,7 @@ class BaseQueue {
    HsaQueueResource m_Resources;
    HsaMemoryBuffer *m_QueueBuf;
    unsigned int m_Node;
+    unsigned int m_FamilyId;

    // @return Write pointer modulo queue size in dwords
    virtual unsigned int Wptr() = 0;
@@ -28,6 +28,8 @@
 #include "asic_reg/gfx_7_2_d.h"
 #include "asic_reg/gfx_7_2_sh_mask.h"

+#include "KFDBaseComponentTest.hpp"
+
 Dispatch::Dispatch(const HsaMemoryBuffer& isaBuf, const bool eventAutoReset)
    :m_IsaBuf(isaBuf), m_IndirectBuf(PACKETTYPE_PM4, PAGE_SIZE / sizeof(unsigned int), isaBuf.Node()),
    m_DimX(1), m_DimY(1), m_DimZ(1), m_pArg1(NULL), m_pArg2(NULL), m_pEop(NULL), m_ScratchEn(false),
@@ -39,6 +41,8 @@ Dispatch::Dispatch(const HsaMemoryBuffer& isaBuf, const bool eventAutoReset)
    eventDesc.SyncVar.SyncVarSize = 0;

    hsaKmtCreateEvent(&eventDesc, !eventAutoReset, false, &m_pEop);
+
+    m_FamilyId  = g_baseTest->GetFamilyIdFromNodeId(isaBuf.Node());
 }

 Dispatch::~Dispatch() {
@@ -69,6 +73,7 @@ void Dispatch::SetSpiPriority(unsigned int priority) {

 void Dispatch::Submit(BaseQueue& queue) {
    ASSERT_NE(m_pEop, (void*)0);
+    EXPECT_EQ(m_FamilyId, queue.GetFamilyId());

    BuildIb();

@@ -80,7 +85,7 @@ void Dispatch::Submit(BaseQueue& queue) {
            EventData.EventData.SyncVar.SyncVar.UserData, m_pEop->EventId));
    }

-    queue.PlaceAndSubmitPacket(PM4ReleaseMemoryPacket(g_TestGPUFamilyId, false, m_pEop->EventData.HWData2, m_pEop->EventId));
+    queue.PlaceAndSubmitPacket(PM4ReleaseMemoryPacket(m_FamilyId, false, m_pEop->EventData.HWData2, m_pEop->EventId));

    if (!queue.GetSkipWaitConsump())
        queue.Wait4PacketConsumption();
@@ -200,14 +205,14 @@ void Dispatch::BuildIb() {
    // ORDERED_APPEND_MODE=0, USE_THREAD_DIMENSIONS=1, ORDER_MODE=0, DISPATCH_CACHE_CNTL=0,
    // SCALAR_L1_INV_VOL=0, VECTOR_L1_INV_VOL=0, DATA_ATC=?, RESTORE=0}

-    m_IndirectBuf.AddPacket(PM4AcquireMemoryPacket(g_TestGPUFamilyId));
+    m_IndirectBuf.AddPacket(PM4AcquireMemoryPacket(m_FamilyId));

    m_IndirectBuf.AddPacket(PM4SetShaderRegPacket(mmCOMPUTE_START_X, COMPUTE_DISPATCH_DIMS_VALUES,
                                                  ARRAY_SIZE(COMPUTE_DISPATCH_DIMS_VALUES)));

    m_IndirectBuf.AddPacket(PM4SetShaderRegPacket(mmCOMPUTE_PGM_LO,
-        (g_TestGPUFamilyId >= FAMILY_AI) ? COMPUTE_PGM_VALUES_GFX9 : COMPUTE_PGM_VALUES_GFX8,
-        (g_TestGPUFamilyId >= FAMILY_AI) ? ARRAY_SIZE(COMPUTE_PGM_VALUES_GFX9) : ARRAY_SIZE(COMPUTE_PGM_VALUES_GFX8)));
+        (m_FamilyId >= FAMILY_AI) ? COMPUTE_PGM_VALUES_GFX9 : COMPUTE_PGM_VALUES_GFX8,
+        (m_FamilyId >= FAMILY_AI) ? ARRAY_SIZE(COMPUTE_PGM_VALUES_GFX9) : ARRAY_SIZE(COMPUTE_PGM_VALUES_GFX8)));
    m_IndirectBuf.AddPacket(PM4SetShaderRegPacket(mmCOMPUTE_PGM_RSRC1, COMPUTE_PGM_RSRC,
                                                  ARRAY_SIZE(COMPUTE_PGM_RSRC)));

@@ -225,5 +230,5 @@ void Dispatch::BuildIb() {

    m_IndirectBuf.AddPacket(PM4PartialFlushPacket());

-    m_IndirectBuf.AddPacket(PM4AcquireMemoryPacket(g_TestGPUFamilyId));
+    m_IndirectBuf.AddPacket(PM4AcquireMemoryPacket(m_FamilyId));
 }
@@ -70,6 +70,7 @@ class Dispatch {

    HSAuint64  m_scratch_base;
    unsigned int m_SpiPriority;
+    unsigned int  m_FamilyId;
 };

 #endif  // __KFD_DISPATCH__H__
@@ -61,12 +61,12 @@ void KFDBaseComponentTest::SetUp() {
    const HsaNodeProperties *nodeProperties = m_NodeInfo.HsaDefaultGPUNodeProperties();
    ASSERT_NOTNULL(nodeProperties) << "failed to get HSA default GPU Node properties";

-    g_TestGPUFamilyId = FamilyIdFromNode(nodeProperties);
+    m_FamilyId = FamilyIdFromNode(nodeProperties);

    GetSdmaInfo(nodeProperties, &m_numSdmaEngines, &m_numSdmaXgmiEngines,
                    &m_numSdmaQueuesPerEngine);

-    m_FamilyId = g_TestGPUFamilyId;
+    g_baseTest = this;

    ROUTINE_END
 }
@@ -84,6 +84,7 @@ void KFDBaseComponentTest::TearDown() {

    EXPECT_SUCCESS(hsaKmtReleaseSystemProperties());
    EXPECT_SUCCESS(hsaKmtCloseKFD());
+    g_baseTest = NULL;

    ROUTINE_END
 }
@@ -128,6 +129,11 @@ HSAuint64 KFDBaseComponentTest::GetVramSize(int defaultGPUNode) {
    return 0;
 }

+unsigned int KFDBaseComponentTest::GetFamilyIdFromNodeId(unsigned int nodeId)
+{
+    return  FamilyIdFromNode(m_NodeInfo.GetNodeProperties(nodeId));
+}
+
 int KFDBaseComponentTest::FindDRMRenderNode(int gpuNode) {
    HsaNodeProperties *nodeProperties;
    _HSAKMT_STATUS status;
@@ -55,6 +55,8 @@ class KFDBaseComponentTest : public testing::Test {
 // @brief Finds DRM Render node corresponding to gpuNode
 // @return DRM Render Node if successful or -1 on failure
    int FindDRMRenderNode(int gpuNode);
+    unsigned int GetFamilyIdFromNodeId(unsigned int nodeId);
+    unsigned int GetFamilyIdFromDefaultNode(){ return m_FamilyId; }

 protected:
    HsaVersionInfo  m_VersionInfo;
@@ -76,4 +78,5 @@ class KFDBaseComponentTest : public testing::Test {
    virtual void TearDown();
 };

+extern KFDBaseComponentTest* g_baseTest;
 #endif  //  __KFD_BASE_COMPONENT_TEST__H__
@@ -147,8 +147,9 @@ class QueueAndSignalBenchmark {
        uint64_t startTime;
        PM4Queue queue;

+        unsigned int familyId = g_baseTest->GetFamilyIdFromNodeId(node);
        HsaEvent** pHsaEvent = reinterpret_cast<HsaEvent**>(calloc(eventCount, sizeof(HsaEvent*)));
-        size_t packetSize = PM4ReleaseMemoryPacket(g_TestGPUFamilyId, false, 0, 0).SizeInBytes();
+        size_t packetSize = PM4ReleaseMemoryPacket(familyId, false, 0, 0).SizeInBytes();
        int qSize = fmax(PAGE_SIZE, pow2_round_up(packetSize*eventCount + 1));

        time = 0;
@@ -162,7 +163,7 @@ class QueueAndSignalBenchmark {
            if (r != HSAKMT_STATUS_SUCCESS)
                goto exit;

-            queue.PlacePacket(PM4ReleaseMemoryPacket(g_TestGPUFamilyId, false, pHsaEvent[i]->EventData.HWData2, pHsaEvent[i]->EventId));
+            queue.PlacePacket(PM4ReleaseMemoryPacket(familyId, false, pHsaEvent[i]->EventData.HWData2, pHsaEvent[i]->EventId));
        }

        startTime = gettime();
@@ -30,7 +30,6 @@ extern unsigned int g_TestTimeOut;
 extern int g_TestNodeId;
 extern int g_TestDstNodeId;
 extern bool g_IsChildProcess;
-extern unsigned int g_TestGPUFamilyId;

 // Each test should call TEST_START with the test custom profile and HW scheduling
 enum TESTPROFILE{
@@ -54,6 +54,7 @@ int g_TestNodeId;
 int g_TestDstNodeId;
 bool g_IsChildProcess;
 unsigned int g_TestGPUFamilyId;
+class KFDBaseComponentTest *g_baseTest;

 GTEST_API_ int main(int argc, char **argv) {
    // Default values for run parameters
@@ -103,7 +103,7 @@ class AsyncMPSQ {
            if (m_queueType == HSA_QUEUE_SDMA)
                return SDMATimePacket(0).SizeInBytes();
            else if (m_queueType == HSA_QUEUE_COMPUTE)
-                return PM4ReleaseMemoryPacket(g_TestGPUFamilyId, 0, 0, 0, 0, 0).SizeInBytes();
+                return PM4ReleaseMemoryPacket(m_queue->GetFamilyId(), 0, 0, 0, 0, 0).SizeInBytes();
            return 0;
        }

@@ -126,7 +126,7 @@ class AsyncMPSQ {
                PlacePacket(SDMATimePacket(addr));
            else if (m_queueType == HSA_QUEUE_COMPUTE)
                PlacePacket(
-                        PM4ReleaseMemoryPacket(g_TestGPUFamilyId, true, (HSAuint64)addr, 0, true, true));
+                        PM4ReleaseMemoryPacket(m_queue->GetFamilyId(), true, (HSAuint64)addr, 0, true, true));
            else
                WARN() << "Unsupported queue type!" << std::endl;
        }
@@ -26,6 +26,7 @@
 #include <string.h>
 #include "PM4Packet.hpp"
 #include "hsakmttypes.h"
+#include "KFDBaseComponentTest.hpp"

 #include "asic_reg/gfx_7_2_enum.h"

@@ -75,7 +76,7 @@ void PM4WriteDataPacket::InitPacket(unsigned int *destBuf, void *data) {

 PM4ReleaseMemoryPacket::PM4ReleaseMemoryPacket(unsigned int familyId, bool isPolling,
                    uint64_t address, uint64_t data, bool is64bit, bool isTimeStamp):m_pPacketData(NULL) {
-
+    m_FamilyId = familyId;
    if (familyId < FAMILY_AI)
        InitPacketCI(isPolling, address, data, is64bit, isTimeStamp);
    else if (familyId < FAMILY_NV)
@@ -264,6 +265,7 @@ void PM4IndirectBufPacket::InitPacket(IndirectBuffer *pIb) {
 }
 PM4AcquireMemoryPacket::PM4AcquireMemoryPacket(unsigned int familyId):m_pPacketData(NULL)
 {
+    m_FamilyId = familyId;

    if (familyId < FAMILY_NV)
        InitPacketAI();
@@ -278,6 +280,7 @@ void PM4AcquireMemoryPacket::InitPacketAI(void) {
    pkt = reinterpret_cast<PM4ACQUIRE_MEM*>(calloc(1, m_packetSize));
    m_pPacketData = pkt;
    EXPECT_NOTNULL(m_pPacketData);
+
    InitPM4Header(pkt->header,  IT_ACQUIRE_MEM);
    pkt->bitfields2.coher_cntl     = 0x28c00000;  // copied from the way the HSART does this.
    pkt->bitfields2.engine         = engine_acquire_mem_PFP_0;
@@ -56,7 +56,7 @@ unsigned int PM4Queue::RptrWhenConsumed() {

 void PM4Queue::SubmitPacket() {
    // m_pending Wptr is in dwords
-    if (g_TestGPUFamilyId < FAMILY_AI) {
+    if (m_FamilyId < FAMILY_AI) {
        // Pre-Vega10 uses 32-bit wptr and doorbell
        MemoryBarrier();
        *m_Resources.Queue_write_ptr = m_pendingWptr;
@@ -73,7 +73,7 @@ void PM4Queue::SubmitPacket() {

 void PM4Queue::Wait4PacketConsumption(HsaEvent *event, unsigned int timeOut) {
    if (event) {
-        PlaceAndSubmitPacket(PM4ReleaseMemoryPacket(g_TestGPUFamilyId, 0,
+        PlaceAndSubmitPacket(PM4ReleaseMemoryPacket(m_FamilyId, 0,
                    event->EventData.HWData2,
                    event->EventId,
                    true));
@@ -30,7 +30,7 @@
 /* Byte/dword count in many SDMA packets is 1-based in AI, meaning a
 * count of 1 is encoded as 0.
 */
-#define SDMA_COUNT(c) (g_TestGPUFamilyId < FAMILY_AI ? (c) : (c)-1)
+#define SDMA_COUNT(c) (m_FamilyId < FAMILY_AI ? (c) : (c)-1)

 SDMAWriteDataPacket::SDMAWriteDataPacket(void):
    packetData(NULL) {
@@ -63,7 +63,7 @@ unsigned int SDMAQueue::RptrWhenConsumed() {

 void SDMAQueue::SubmitPacket() {
    // m_pending Wptr is in dwords
-    if (g_TestGPUFamilyId < FAMILY_AI) {
+    if (m_FamilyId < FAMILY_AI) {
        // Pre-Vega10 uses 32-bit wptr and doorbell
        unsigned int wPtrInBytes = m_pendingWptr * sizeof(unsigned int);
        MemoryBarrier();