kfdtest: Update KFDMemoryTest to LLVM Asm

- Reformat shaders for legibility
- Move assembly processes to from IsaGen (CompileShader) to Assembler
(RunAssembleBuf)
- LLVM syntax change on ScratchCopyDwordIsa_gfx10:
hwreg(HW_REG_SHADER_FLAT_SCRATCH_LO/HI) -> hwreg(HW_REG_FLAT_SCR_LO/HI)
- Fix bug in CopyOnSignalIsa_gfx10 and PollMemoryIsa_gfx10 whereby
flat_store_dword used vector reg format v[n,n]. Changed to v[n:n]

Signed-off-by: Graham Sider <Graham.Sider@amd.com>
Change-Id: Id182cfb8aeb7372366c59affb5cbdd145909ee96
This commit is contained in:
Graham Sider
2021-09-24 18:45:28 -04:00
committed by Harish Kasiviswanathan
parent 235636d598
commit 039bce94a6
+284 -330
View File
@@ -39,166 +39,124 @@
#include "SDMAPacket.hpp"
#include "linux/kfd_ioctl.h"
const char* gfx8_ScratchCopyDword =
"\
shader ScratchCopyDword\n\
asic(VI)\n\
type(CS)\n\
/*copy the parameters from scalar registers to vector registers*/\n\
v_mov_b32 v0, s0\n\
v_mov_b32 v1, s1\n\
v_mov_b32 v2, s2\n\
v_mov_b32 v3, s3\n\
/*set up the scratch parameters. This assumes a single 16-reg block.*/\n\
s_mov_b32 flat_scratch_lo, 8/*2 dwords of scratch per thread*/\n\
s_mov_b32 flat_scratch_hi, 0/*offset in units of 256bytes*/\n\
/*copy a dword between the passed addresses*/\n\
flat_load_dword v4, v[0:1] slc\n\
s_waitcnt vmcnt(0)&lgkmcnt(0)\n\
flat_store_dword v[2:3], v4 slc\n\
\n\
s_endpgm\n\
\n\
end\n\
";
const char* gfx9_ScratchCopyDword =
"\
shader ScratchCopyDword\n\
asic(GFX9)\n\
type(CS)\n\
/*copy the parameters from scalar registers to vector registers*/\n\
v_mov_b32 v0, s0\n\
v_mov_b32 v1, s1\n\
v_mov_b32 v2, s2\n\
v_mov_b32 v3, s3\n\
/*set up the scratch parameters. This assumes a single 16-reg block.*/\n\
s_mov_b32 flat_scratch_lo, s4\n\
s_mov_b32 flat_scratch_hi, s5\n\
/*copy a dword between the passed addresses*/\n\
flat_load_dword v4, v[0:1] slc\n\
s_waitcnt vmcnt(0)&lgkmcnt(0)\n\
flat_store_dword v[2:3], v4 slc\n\
\n\
s_endpgm\n\
\n\
end\n\
";
const char* gfx10_ScratchCopyDword =
"\
shader ScratchCopyDword\n\
asic(GFX10)\n\
type(CS)\n\
wave_size(32)\n\
/*copy the parameters from scalar registers to vector registers*/\n\
v_mov_b32 v0, s0\n\
v_mov_b32 v1, s1\n\
v_mov_b32 v2, s2\n\
v_mov_b32 v3, s3\n\
/*set up the scratch parameters. This assumes a single 16-reg block.*/\n\
s_setreg_b32 hwreg(HW_REG_SHADER_FLAT_SCRATCH_LO), s4\n\
s_setreg_b32 hwreg(HW_REG_SHADER_FLAT_SCRATCH_HI), s5\n\
/*copy a dword between the passed addresses*/\n\
flat_load_dword v4, v[0:1] slc\n\
s_waitcnt vmcnt(0)&lgkmcnt(0)\n\
flat_store_dword v[2:3], v4 slc\n\
\n\
s_endpgm\n\
\n\
end\n\
";
const char* aldbrn_ScratchCopyDword =
"\
shader ScratchCopyDword\n\
asic(ALDEBARAN)\n\
type(CS)\n\
/*copy the parameters from scalar registers to vector registers*/\n\
v_mov_b32 v0, s0\n\
v_mov_b32 v1, s1\n\
v_mov_b32 v2, s2\n\
v_mov_b32 v3, s3\n\
/*set up the scratch parameters. This assumes a single 16-reg block.*/\n\
s_mov_b32 flat_scratch_lo, s4\n\
s_mov_b32 flat_scratch_hi, s5\n\
/*copy a dword between the passed addresses*/\n\
flat_load_dword v4, v[0:1] slc\n\
s_waitcnt vmcnt(0)&lgkmcnt(0)\n\
flat_store_dword v[2:3], v4 slc\n\
\n\
s_endpgm\n\
\n\
end\n\
";
static const char* ScratchCopyDwordIsa_gfx8 = R"(
.text
// Copy the parameters from scalar registers to vector registers
v_mov_b32_e32 v0, s0
v_mov_b32_e32 v1, s1
v_mov_b32_e32 v2, s2
v_mov_b32_e32 v3, s3
// Setup the scratch parameters. This assumes a single 16-reg block
s_mov_b32 flat_scratch_lo, 8
s_mov_b32 flat_scratch_hi, 0
// Copy a dword between the passed addresses
flat_load_dword v4, v[0:1] slc
s_waitcnt vmcnt(0) & lgkmcnt(0)
flat_store_dword v[2:3], v4 slc
s_endpgm
)";
static const char* ScratchCopyDwordIsa_gfx9 = R"(
.text
// Copy the parameters from scalar registers to vector registers
v_mov_b32 v0, s0
v_mov_b32 v1, s1
v_mov_b32 v2, s2
v_mov_b32 v3, s3
// Setup the scratch parameters. This assumes a single 16-reg block
s_mov_b32 flat_scratch_lo, s4
s_mov_b32 flat_scratch_hi, s5
// Copy a dword between the passed addresses
flat_load_dword v4, v[0:1] slc
s_waitcnt vmcnt(0) & lgkmcnt(0)
flat_store_dword v[2:3], v4 slc
s_endpgm
)";
static const char* ScratchCopyDwordIsa_gfx10 = R"(
.text
// Copy the parameters from scalar registers to vector registers
v_mov_b32 v0, s0
v_mov_b32 v1, s1
v_mov_b32 v2, s2
v_mov_b32 v3, s3
// Setup the scratch parameters. This assumes a single 16-reg block
s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s4
s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s5
// Copy a dword between the passed addresses
flat_load_dword v4, v[0:1] slc
s_waitcnt vmcnt(0) & lgkmcnt(0)
flat_store_dword v[2:3], v4 slc
s_endpgm
)";
static const char* ScratchCopyDwordIsa_gfx9aldbrn = R"(
.text
// Copy the parameters from scalar registers to vector registers
v_mov_b32 v0, s0
v_mov_b32 v1, s1
v_mov_b32 v2, s2
v_mov_b32 v3, s3
// Setup the scratch parameters. This assumes a single 16-reg block
s_mov_b32 flat_scratch_lo, s4
s_mov_b32 flat_scratch_hi, s5
// Copy a dword between the passed addresses
flat_load_dword v4, v[0:1] slc
s_waitcnt vmcnt(0) & lgkmcnt(0)
flat_store_dword v[2:3], v4 slc
s_endpgm
)";
/* Continuously poll src buffer and check buffer value
* After src buffer is filled with specific value (0x5678,
* by host program), fill dst buffer with specific
* value(0x5678) and quit
*/
const char* gfx9_PollMemory =
"\
shader ReadMemory\n\
wave_size(32)\n\
type(CS)\n\
/* Assume src address in s0, s1 and dst address in s2, s3*/\n\
s_movk_i32 s18, 0x5678\n\
LOOP:\n\
s_load_dword s16, s[0:1], 0x0 glc\n\
s_cmp_eq_i32 s16, s18\n\
s_cbranch_scc0 LOOP\n\
s_store_dword s18, s[2:3], 0x0 glc\n\
s_endpgm\n\
end\n\
";
static const char* PollMemoryIsa_gfx9 = R"(
.text
// Assume src address in s0, s1, and dst address in s2, s3
s_movk_i32 s18, 0x5678
LOOP:
s_load_dword s16, s[0:1], 0x0 glc
s_cmp_eq_i32 s16, s18
s_cbranch_scc0 LOOP
s_store_dword s18, s[2:3], 0x0 glc
s_endpgm
)";
/* Similar to gfx9_PollMemory except that the buffer
/* Similar to PollMemoryIsa_gfx9 except that the buffer
* polled can be Non-coherant memory. SCC system-level
* cache coherence is not supported in scalar (smem) path.
* Use vmem operations with scc
*/
const char* gfx9_PollNCMemory =
"\
shader ReadMemory\n\
asic(ALDEBARAN)\n\
wave_size(32)\n\
type(CS)\n\
/* Assume src address in s0, s1 and dst address in s2, s3*/\n\
v_mov_b32 v6, 0x5678\n\
v_mov_b32 v0, s0\n\
v_mov_b32 v1, s1\n\
LOOP:\n\
flat_load_dword v4, v[0:1] scc\n\
v_cmp_eq_u32 vcc, v4, v6\n\
s_cbranch_vccz LOOP\n\
v_mov_b32 v0, s2\n\
v_mov_b32 v1, s3\n\
flat_store_dword v[0:1], v6 scc\n\
s_endpgm\n\
end\n\
";
static const char* PollNCMemoryIsa_gfx9 = R"(
.text
// Assume src address in s0, s1, and dst address in s2, s3
v_mov_b32 v6, 0x5678
v_mov_b32 v0, s0
v_mov_b32 v1, s1
LOOP:
flat_load_dword v4, v[0:1] scc
v_cmp_eq_u32 vcc, v4, v6
s_cbranch_vccz LOOP
v_mov_b32 v0, s2
v_mov_b32 v1, s3
flat_store_dword v[0:1], v6 scc
s_endpgm
)";
const char* gfx10_PollMemory =
"\
shader ReadMemory\n\
wave_size(32)\n\
type(CS)\n\
/* Assume src address in s0, s1 and dst address in s2, s3*/\n\
s_movk_i32 s18, 0x5678\n\
v_mov_b32 v0, s2\n\
v_mov_b32 v1, s3\n\
v_mov_b32 v2, 0x5678\n\
LOOP:\n\
s_load_dword s16, s[0:1], 0x0 glc\n\
s_cmp_eq_i32 s16, s18\n\
s_cbranch_scc0 LOOP\n\
flat_store_dword v[0,1], v2 slc\n\
s_waitcnt vmcnt(0)&lgkmcnt(0)\n\
s_endpgm\n\
end\n\
";
static const char* PollMemoryIsa_gfx10 = R"(
.text
// Assume src address in s0, s1, and dst address in s2, s3
s_movk_i32 s18, 0x5678
v_mov_b32 v0, s2
v_mov_b32 v1, s3
v_mov_b32 v2, 0x5678
LOOP:
s_load_dword s16, s[0:1], 0x0 glc
s_cmp_eq_i32 s16, s18
s_cbranch_scc0 LOOP
flat_store_dword v[0:1], v2 slc
s_waitcnt vmcnt(0) & lgkmcnt(0)
s_endpgm
)";
/* Input: A buffer of at least 3 dwords.
* DW0: used as a signal. 0xcafe means it is signaled
@@ -209,119 +167,99 @@ type(CS)\n\
* Once signal buffer is signaled, it copies input buffer
* to output buffer
*/
const char* gfx9_CopyOnSignal =
"\
shader CopyOnSignal\n\
wave_size(32)\n\
type(CS)\n\
/* Assume input buffer in s0, s1 */\n\
s_mov_b32 s18, 0xcafe\n\
POLLSIGNAL:\n\
s_load_dword s16, s[0:1], 0x0 glc\n\
s_cmp_eq_i32 s16, s18\n\
s_cbranch_scc0 POLLSIGNAL\n\
s_load_dword s17, s[0:1], 0x4 glc\n\
s_waitcnt vmcnt(0) & lgkmcnt(0)\n\
s_store_dword s17, s[0:1], 0x8 glc\n\
s_waitcnt vmcnt(0) & lgkmcnt(0)\n\
s_endpgm\n\
end\n\
";
static const char* CopyOnSignalIsa_gfx9 = R"(
.text
// Assume input buffer in s0, s1
s_mov_b32 s18, 0xcafe
POLLSIGNAL:
s_load_dword s16, s[0:1], 0x0 glc
s_cmp_eq_i32 s16, s18
s_cbranch_scc0 POLLSIGNAL
s_load_dword s17, s[0:1], 0x4 glc
s_waitcnt vmcnt(0) & lgkmcnt(0)
s_store_dword s17, s[0:1], 0x8 glc
s_waitcnt vmcnt(0) & lgkmcnt(0)
s_endpgm
)";
const char* gfx10_CopyOnSignal =
"\
shader CopyOnSignal\n\
wave_size(32)\n\
type(CS)\n\
/* Assume input buffer in s0, s1 */\n\
s_add_u32 s2, s0, 0x8\n\
s_addc_u32 s3, s1, 0x0\n\
s_mov_b32 s18, 0xcafe\n\
v_mov_b32 v0, s0\n\
v_mov_b32 v1, s1\n\
v_mov_b32 v4, s2\n\
v_mov_b32 v5, s3\n\
POLLSIGNAL:\n\
s_load_dword s16, s[0:1], 0x0 glc\n\
s_cmp_eq_i32 s16, s18\n\
s_cbranch_scc0 POLLSIGNAL\n\
s_load_dword s17, s[0:1], 0x4 glc\n\
s_waitcnt vmcnt(0) & lgkmcnt(0)\n\
v_mov_b32 v2, s17\n\
flat_store_dword v[4,5], v2 glc\n\
s_waitcnt vmcnt(0) & lgkmcnt(0)\n\
s_endpgm\n\
end\n\
";
static const char* CopyOnSignalIsa_gfx10 = R"(
.text
// Assume input buffer in s0, s1
s_add_u32 s2, s0, 0x8
s_addc_u32 s3, s1, 0x0
s_mov_b32 s18, 0xcafe
v_mov_b32 v0, s0
v_mov_b32 v1, s1
v_mov_b32 v4, s2
v_mov_b32 v5, s3
POLLSIGNAL:
s_load_dword s16, s[0:1], 0x0 glc
s_cmp_eq_i32 s16, s18
s_cbranch_scc0 POLLSIGNAL
s_load_dword s17, s[0:1], 0x4 glc
s_waitcnt vmcnt(0) & lgkmcnt(0)
v_mov_b32 v2, s17
flat_store_dword v[4:5], v2 glc
s_waitcnt vmcnt(0) & lgkmcnt(0)
s_endpgm
)";
/* Input0: A buffer of at least 2 dwords.
* DW0: used as a signal. Write 0xcafe to signal
* DW1: Write to this buffer for other device to read.
* Input1: mmio base address
*/
const char* gfx9_WriteAndSignal =
"\
shader WriteAndSignal\n\
wave_size(32)\n\
type(CS)\n\
/* Assume input buffer in s0, s1 */\n\
s_mov_b32 s18, 0xbeef\n\
s_store_dword s18, s[0:1], 0x4 glc\n\
s_mov_b32 s18, 0x1\n\
s_store_dword s18, s[2:3], 0 glc\n\
s_mov_b32 s18, 0xcafe\n\
s_store_dword s18, s[0:1], 0x0 glc\n\
s_endpgm\n\
end\n\
";
static const char* WriteAndSignalIsa_gfx9 = R"(
.text
// Assume input buffer in s0, s1
s_mov_b32 s18, 0xbeef
s_store_dword s18, s[0:1], 0x4 glc
s_mov_b32 s18, 0x1
s_store_dword s18, s[2:3], 0 glc
s_mov_b32 s18, 0xcafe
s_store_dword s18, s[0:1], 0x0 glc
s_endpgm
)";
/* Continuously poll the flag at src buffer
* After the flag of s[0:1] is 1 filled,
* copy the value from s[0:1]+4 to dst buffer
*/
const char* gfx9_PollAndCopy =
"\
shader CopyMemory\n\
wave_size(32)\n\
type(CS)\n\
/* Assume src buffer in s[0:1] and dst buffer in s[2:3]*/\n\
s_movk_i32 s18, 0x1\n\
LOOP:\n\
s_load_dword s16, s[0:1], 0x0 glc\n\
s_cmp_eq_i32 s16, s18\n\
s_cbranch_scc0 LOOP\n\
s_load_dword s17, s[0:1], 0x4 glc\n\
s_waitcnt vmcnt(0) & lgkmcnt(0)\n\
s_store_dword s17, s[2:3], 0x0 glc:1\n\
s_waitcnt vmcnt(0) & lgkmcnt(0)\n\
s_endpgm\n\
end\n\
";
static const char* PollAndCopyIsa_gfx9 = R"(
.text
// Assume src buffer in s[0:1] and dst buffer in s[2:3]
s_movk_i32 s18, 0x1
LOOP:
s_load_dword s16, s[0:1], 0x0 glc
s_cmp_eq_i32 s16, s18
s_cbranch_scc0 LOOP
s_load_dword s17, s[0:1], 0x4 glc
s_waitcnt vmcnt(0) & lgkmcnt(0)
s_store_dword s17, s[2:3], 0x0 glc
s_waitcnt vmcnt(0) & lgkmcnt(0)
s_endpgm
)";
const char* gfx9aldbrn_PollAndCopy =
"\
shader CopyMemory\n\
wave_size(32)\n\
type(CS)\n\
/* Assume src buffer in s[0:1] and dst buffer in s[2:3]*/\n\
v_mov_b32 v0, s0\n\
v_mov_b32 v1, s1\n\
v_mov_b32 v18, 0x1\n\
LOOP:\n\
flat_load_dword v16, v[0:1] glc\n\
s_waitcnt vmcnt(0) & lgkmcnt(0)\n\
v_cmp_eq_i32 vcc, v16, v18\n\
s_cbranch_vccz LOOP\n\
buffer_invl2\n\
s_load_dword s17, s[0:1], 0x4 glc\n\
s_waitcnt vmcnt(0) & lgkmcnt(0)\n\
s_store_dword s17, s[2:3], 0x0 glc\n\
s_waitcnt vmcnt(0) & lgkmcnt(0)\n\
buffer_wbl2\n\
s_waitcnt vmcnt(0) & lgkmcnt(0)\n\
s_endpgm\n\
end\n\
";
static const char* PollAndCopyIsa_gfx9aldbrn = R"(
.text
// Assume src buffer in s[0:1] and dst buffer in s[2:3]
v_mov_b32 v0, s0
v_mov_b32 v1, s1
v_mov_b32 v18, 0x1
LOOP:
flat_load_dword v16, v[0:1] glc
s_waitcnt vmcnt(0) & lgkmcnt(0)
v_cmp_eq_i32 vcc, v16, v18
s_cbranch_vccz LOOP
buffer_invl2
s_load_dword s17, s[0:1], 0x4 glc
s_waitcnt vmcnt(0) & lgkmcnt(0)
s_store_dword s17, s[2:3], 0x0 glc
s_waitcnt vmcnt(0) & lgkmcnt(0)
buffer_wbl2
s_waitcnt vmcnt(0) & lgkmcnt(0)
s_endpgm
)";
/* Input0: A buffer of at least 2 dwords.
* DW0: used as a signal. Write 0x1 to signal
@@ -330,51 +268,45 @@ type(CS)\n\
* Input1: A buffer of at least 2 dwords.
* DW0: used as the value to be written.
*/
const char* gfx9aldbrn_WriteFlagAndValue =
"\
shader WriteMemory\n\
wave_size(32)\n\
type(CS)\n\
/* Assume two inputs buffer in s[0:1] and s[2:3]*/\n\
v_mov_b32 v0, s0\n\
v_mov_b32 v1, s1\n\
s_load_dword s18, s[2:3], 0x0 glc\n\
s_waitcnt vmcnt(0) & lgkmcnt(0)\n\
s_store_dword s18, s[0:1], 0x4 glc\n\
s_waitcnt vmcnt(0) & lgkmcnt(0)\n\
buffer_wbl2\n\
s_waitcnt vmcnt(0) & lgkmcnt(0)\n\
v_mov_b32 v16, 0x1\n\
flat_store_dword v[0:1], v16 glc\n\
s_endpgm\n\
end\n\
";
static const char* WriteFlagAndValueIsa_gfx9aldbrn = R"(
.text
// Assume two inputs buffer in s[0:1] and s[2:3]
v_mov_b32 v0, s0
v_mov_b32 v1, s1
s_load_dword s18, s[2:3], 0x0 glc
s_waitcnt vmcnt(0) & lgkmcnt(0)
s_store_dword s18, s[0:1], 0x4 glc
s_waitcnt vmcnt(0) & lgkmcnt(0)
buffer_wbl2
s_waitcnt vmcnt(0) & lgkmcnt(0)
v_mov_b32 v16, 0x1
flat_store_dword v[0:1], v16 glc
s_endpgm
)";
const char* gfx10_WriteAndSignal =
"\
shader WriteAndSignal\n\
wave_size(32)\n\
type(CS)\n\
/* Assume input buffer in s0, s1 */\n\
s_add_u32 s4, s0, 0x4\n\
s_addc_u32 s5, s1, 0x0\n\
v_mov_b32 v0, s0\n\
v_mov_b32 v1, s1\n\
v_mov_b32 v2, s2\n\
v_mov_b32 v3, s3\n\
v_mov_b32 v4, s4\n\
v_mov_b32 v5, s5\n\
v_mov_b32 v18, 0xbeef\n\
flat_store_dword v[4:5], v18 glc\n\
v_mov_b32 v18, 0x1\n\
flat_store_dword v[2:3], v18 glc\n\
v_mov_b32 v18, 0xcafe\n\
flat_store_dword v[0:1], v18 glc\n\
s_endpgm\n\
end\n\
";
static const char* WriteAndSignalIsa_gfx10 = R"(
.text
// Assume input buffer in s0, s1
s_add_u32 s4, s0, 0x4
s_addc_u32 s5, s1, 0x0
v_mov_b32 v0, s0
v_mov_b32 v1, s1
v_mov_b32 v2, s2
v_mov_b32 v3, s3
v_mov_b32 v4, s4
v_mov_b32 v5, s5
v_mov_b32 v18, 0xbeef
flat_store_dword v[4:5], v18 glc
v_mov_b32 v18, 0x1
flat_store_dword v[2:3], v18 glc
v_mov_b32 v18, 0xcafe
flat_store_dword v[0:1], v18 glc
s_endpgm
)";
//These gfx9_PullMemory, gfx9_CopyOnSignal, gfx9_WriteAndSignal shaders can be used by both gfx9 and gfx10
/* These PollMemoryIsa_gfx9, CopyOnSignalIsa_gfx9,
* WriteAndSignalIsa_gfx9 shaders can be used by both gfx9 and gfx10
*/
void KFDMemoryTest::SetUp() {
ROUTINE_START
@@ -508,16 +440,15 @@ TEST_F(KFDMemoryTest, MapUnmapToNodes) {
HsaMemoryBuffer dstBuffer(PAGE_SIZE, defaultGPUNode);
const char *pReadMemory;
if (m_FamilyId < FAMILY_NV)
pReadMemory = gfx9_PollMemory;
else
pReadMemory = gfx10_PollMemory;
if (m_NodeInfo.IsNodeXGMItoCPU(defaultGPUNode))
/* On A+A system memory is mapped as NC */
m_pIsaGen->CompileShader(gfx9_PollNCMemory, "ReadMemory", isaBuffer);
pReadMemory = PollNCMemoryIsa_gfx9;
else if (m_FamilyId < FAMILY_NV)
pReadMemory = PollMemoryIsa_gfx9;
else
m_pIsaGen->CompileShader(pReadMemory, "ReadMemory", isaBuffer);
pReadMemory = PollMemoryIsa_gfx10;
ASSERT_SUCCESS(m_pAsm->RunAssembleBuf(pReadMemory, isaBuffer.As<char*>()));
PM4Queue pm4Queue;
ASSERT_SUCCESS(pm4Queue.Create(defaultGPUNode));
@@ -855,16 +786,17 @@ TEST_F(KFDMemoryTest, FlatScratchAccess) {
// Initialize the srcBuffer to some fixed value
srcMemBuffer.Fill(0x01010101);
const char *pScratchCopyDword;
const char *pScratchCopyDwordIsa;
if (m_FamilyId < FAMILY_AI)
pScratchCopyDword = gfx8_ScratchCopyDword;
pScratchCopyDwordIsa = ScratchCopyDwordIsa_gfx8;
else if (m_FamilyId < FAMILY_AL)
pScratchCopyDword = gfx9_ScratchCopyDword;
pScratchCopyDwordIsa = ScratchCopyDwordIsa_gfx9;
else if (m_FamilyId == FAMILY_AL)
pScratchCopyDword = aldbrn_ScratchCopyDword;
pScratchCopyDwordIsa = ScratchCopyDwordIsa_gfx9aldbrn;
else
pScratchCopyDword = gfx10_ScratchCopyDword;
m_pIsaGen->CompileShader(pScratchCopyDword, "ScratchCopyDword", isaBuffer);
pScratchCopyDwordIsa = ScratchCopyDwordIsa_gfx10;
ASSERT_SUCCESS(m_pAsm->RunAssembleBuf(pScratchCopyDwordIsa, isaBuffer.As<char*>()));
const HsaNodeProperties *pNodeProperties = m_NodeInfo.GetNodeProperties(defaultGPUNode);
@@ -1728,17 +1660,18 @@ TEST_F(KFDMemoryTest, PtraceAccessInvisibleVram) {
// dstBuffer is cpu accessible gtt memory
HsaMemoryBuffer dstBuffer(PAGE_SIZE, defaultGPUNode);
const char *pScratchCopyDword;
const char *pScratchCopyDwordIsa;
if (m_FamilyId < FAMILY_AI)
pScratchCopyDword = gfx8_ScratchCopyDword;
pScratchCopyDwordIsa = ScratchCopyDwordIsa_gfx8;
else if (m_FamilyId < FAMILY_AL)
pScratchCopyDword = gfx9_ScratchCopyDword;
pScratchCopyDwordIsa = ScratchCopyDwordIsa_gfx9;
else if (m_FamilyId == FAMILY_AL)
pScratchCopyDword = aldbrn_ScratchCopyDword;
pScratchCopyDwordIsa = ScratchCopyDwordIsa_gfx9aldbrn;
else
pScratchCopyDword = gfx10_ScratchCopyDword;
pScratchCopyDwordIsa = ScratchCopyDwordIsa_gfx10;
ASSERT_SUCCESS(m_pAsm->RunAssembleBuf(pScratchCopyDwordIsa, isaBuffer.As<char*>()));
m_pIsaGen->CompileShader(pScratchCopyDword, "ScratchCopyDword", isaBuffer);
Dispatch dispatch0(isaBuffer);
dispatch0.SetArgs(mem0, dstBuffer.As<void*>());
dispatch0.Submit(queue);
@@ -2109,12 +2042,14 @@ TEST_F(KFDMemoryTest, HostHdpFlush) {
PM4Queue queue;
ASSERT_SUCCESS(queue.Create(defaultGPUNode));
HsaMemoryBuffer isaBuffer(PAGE_SIZE, defaultGPUNode, true/*zero*/, false/*local*/, true/*exec*/);
const char *pCopyOnSignal;
const char *pCopyOnSignalIsa;
if (m_FamilyId < FAMILY_NV)
pCopyOnSignal = gfx9_CopyOnSignal;
pCopyOnSignalIsa = CopyOnSignalIsa_gfx9;
else
pCopyOnSignal = gfx10_CopyOnSignal;
m_pIsaGen->CompileShader(pCopyOnSignal, "CopyOnSignal", isaBuffer);
pCopyOnSignalIsa = CopyOnSignalIsa_gfx10;
ASSERT_SUCCESS(m_pAsm->RunAssembleBuf(pCopyOnSignalIsa, isaBuffer.As<char*>()));
Dispatch dispatch0(isaBuffer);
dispatch0.SetArgs(buffer, NULL);
dispatch0.Submit(queue);
@@ -2234,12 +2169,14 @@ TEST_F(KFDMemoryTest, DeviceHdpFlush) {
PM4Queue queue;
ASSERT_SUCCESS(queue.Create(nodes[0]));
HsaMemoryBuffer isaBuffer(PAGE_SIZE, nodes[0], true/*zero*/, false/*local*/, true/*exec*/);
const char *pCopyOnSignal;
const char *pCopyOnSignalIsa;
if (m_FamilyId < FAMILY_NV)
pCopyOnSignal = gfx9_CopyOnSignal;
pCopyOnSignalIsa = CopyOnSignalIsa_gfx9;
else
pCopyOnSignal = gfx10_CopyOnSignal;
m_pIsaGen->CompileShader(pCopyOnSignal, "CopyOnSignal", isaBuffer);
pCopyOnSignalIsa = CopyOnSignalIsa_gfx10;
ASSERT_SUCCESS(m_pAsm->RunAssembleBuf(pCopyOnSignalIsa, isaBuffer.As<char*>()));
Dispatch dispatch(isaBuffer);
dispatch.SetArgs(buffer, NULL);
dispatch.Submit(queue);
@@ -2247,12 +2184,14 @@ TEST_F(KFDMemoryTest, DeviceHdpFlush) {
PM4Queue queue0;
ASSERT_SUCCESS(queue0.Create(nodes[1]));
HsaMemoryBuffer isaBuffer0(PAGE_SIZE, nodes[1], true/*zero*/, false/*local*/, true/*exec*/);
const char *pWriteAndSignal;
const char *pWriteAndSignalIsa;
if (m_FamilyId < FAMILY_NV)
pWriteAndSignal = gfx9_WriteAndSignal;
pWriteAndSignalIsa = WriteAndSignalIsa_gfx9;
else
pWriteAndSignal = gfx10_WriteAndSignal;
m_pIsaGen->CompileShader(pWriteAndSignal, "WriteAndSignal", isaBuffer0);
pWriteAndSignalIsa = WriteAndSignalIsa_gfx10;
ASSERT_SUCCESS(m_pAsm->RunAssembleBuf(pWriteAndSignalIsa, isaBuffer.As<char*>()));
Dispatch dispatch0(isaBuffer0);
dispatch0.SetArgs(buffer, mmioBase);
dispatch0.Submit(queue0);
@@ -2304,7 +2243,9 @@ TEST_F(KFDMemoryTest, CacheInvalidateOnSdmaWrite) {
PM4Queue queue;
ASSERT_SUCCESS(queue.Create(defaultGPUNode));
HsaMemoryBuffer isaBuffer(PAGE_SIZE, defaultGPUNode, true/*zero*/, false/*local*/, true/*exec*/);
m_pIsaGen->CompileShader(gfx9_PollMemory, "ReadMemory", isaBuffer);
ASSERT_SUCCESS(m_pAsm->RunAssembleBuf(PollMemoryIsa_gfx9, isaBuffer.As<char*>()));
Dispatch dispatch(isaBuffer);
dispatch.SetArgs(buffer.As<int*>(), buffer.As<int*>()+dwLocation);
dispatch.Submit(queue);
@@ -2357,7 +2298,9 @@ TEST_F(KFDMemoryTest, CacheInvalidateOnCPUWrite) {
PM4Queue queue;
ASSERT_SUCCESS(queue.Create(defaultGPUNode));
HsaMemoryBuffer isaBuffer(PAGE_SIZE, defaultGPUNode, true/*zero*/, false/*local*/, true/*exec*/);
m_pIsaGen->CompileShader(gfx9_PollMemory, "ReadMemory", isaBuffer);
ASSERT_SUCCESS(m_pAsm->RunAssembleBuf(PollMemoryIsa_gfx9, isaBuffer.As<char*>()));
Dispatch dispatch(isaBuffer);
dispatch.SetArgs(buffer, buffer+100);
dispatch.Submit(queue);
@@ -2419,7 +2362,9 @@ TEST_F(KFDMemoryTest, CacheInvalidateOnRemoteWrite) {
PM4Queue queue;
ASSERT_SUCCESS(queue.Create(defaultGPUNode));
HsaMemoryBuffer isaBuffer(PAGE_SIZE, defaultGPUNode, true/*zero*/, false/*local*/, true/*exec*/);
m_pIsaGen->CompileShader(gfx9_PollMemory, "ReadMemory", isaBuffer);
ASSERT_SUCCESS(m_pAsm->RunAssembleBuf(PollMemoryIsa_gfx9, isaBuffer.As<char*>()));
Dispatch dispatch(isaBuffer);
dispatch.SetArgs(buffer.As<int*>(), buffer.As<int*>()+dwLocation);
dispatch.Submit(queue);
@@ -2500,7 +2445,9 @@ TEST_F(KFDMemoryTest, VramCacheCoherenceWithRemoteGPU) {
PM4Queue queue;
ASSERT_SUCCESS(queue.Create(defaultGPUNode));
HsaMemoryBuffer isaBuffer(PAGE_SIZE, defaultGPUNode, true/*zero*/, false/*local*/, true/*exec*/);
m_pIsaGen->CompileShader(gfx9aldbrn_PollAndCopy, "CopyMemory", isaBuffer);
ASSERT_SUCCESS(m_pAsm->RunAssembleBuf(PollAndCopyIsa_gfx9aldbrn, isaBuffer.As<char*>()));
Dispatch dispatch(isaBuffer);
dispatch.SetArgs(buffer.As<char *>(), buffer.As<char *>()+dwLocation);
dispatch.Submit(queue);
@@ -2515,7 +2462,9 @@ TEST_F(KFDMemoryTest, VramCacheCoherenceWithRemoteGPU) {
PM4Queue queue1;
ASSERT_SUCCESS(queue1.Create(nondefaultNode));
HsaMemoryBuffer isaBuffer1(PAGE_SIZE, nondefaultNode, true/*zero*/, false/*local*/, true/*exec*/);
m_pIsaGen->CompileShader(gfx9aldbrn_WriteFlagAndValue, "WriteMemory", isaBuffer1);
ASSERT_SUCCESS(m_pAsm->RunAssembleBuf(WriteFlagAndValueIsa_gfx9aldbrn, isaBuffer.As<char*>()));
Dispatch dispatch1(isaBuffer1);
dispatch1.SetArgs(buffer.As<char *>(), buffer.As<char *>()+dwSource);
dispatch1.Submit(queue1);
@@ -2569,7 +2518,9 @@ TEST_F(KFDMemoryTest, VramCacheCoherenceWithCPU) {
PM4Queue queue;
ASSERT_SUCCESS(queue.Create(defaultGPUNode));
HsaMemoryBuffer isaBuffer(PAGE_SIZE, defaultGPUNode, true/*zero*/, false/*local*/, true/*exec*/);
m_pIsaGen->CompileShader(gfx9aldbrn_PollAndCopy, "CopyMemory", isaBuffer);
ASSERT_SUCCESS(m_pAsm->RunAssembleBuf(PollAndCopyIsa_gfx9aldbrn, isaBuffer.As<char*>()));
Dispatch dispatch(isaBuffer);
dispatch.SetArgs(buffer, buffer+dwLocation);
dispatch.Submit(queue);
@@ -2627,10 +2578,13 @@ TEST_F(KFDMemoryTest, SramCacheCoherenceWithGPU) {
ASSERT_SUCCESS(queue.Create(defaultGPUNode));
HsaMemoryBuffer isaBuffer(PAGE_SIZE, defaultGPUNode, true/*zero*/, false/*local*/, true/*exec*/);
const char* pPollAndCopyIsa;
if (m_NodeInfo.IsNodeXGMItoCPU(defaultGPUNode))
m_pIsaGen->CompileShader(gfx9aldbrn_PollAndCopy, "CopyMemory", isaBuffer);
pPollAndCopyIsa = PollAndCopyIsa_gfx9aldbrn;
else
m_pIsaGen->CompileShader(gfx9_PollAndCopy, "CopyMemory", isaBuffer);
pPollAndCopyIsa = PollAndCopyIsa_gfx9;
ASSERT_SUCCESS(m_pAsm->RunAssembleBuf(pPollAndCopyIsa, isaBuffer.As<char*>()));
Dispatch dispatch(isaBuffer);
dispatch.SetArgs(fineBuffer, fineBuffer+dwLocation);