AI/rocm-systems
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Implement SDMA_PKT_COPY_LINEAR_RECT for GFX12

Browse Source 
Packet for GFX12 is incompatible with pre-GFX12 as some fields changed
location.   Implement code path and packet specific to GFX12.
This fixes some issues with SDMA blits and 3D images.

Signed-off-by: David Belanger <david.belanger@amd.com>
Change-Id: I56c204aaa12160e563ec960bd3b226cfa94e142d
Signed-off-by: Chris Freehill <cfreehil@amd.com>


[ROCm/ROCR-Runtime commit: 6d147dd3b1]
This commit is contained in:
David Belanger
2024-03-15 21:10:08 -04:00
committed by Chris Freehill
parent f5d734fcf4
commit 36fa572530
2 changed files with 175 additions and 26 deletions
Download Patch File Download Diff File Expand all files Collapse all files
projects/rocr-runtime/runtime/hsa-runtime/core/inc/sdma_registers.h
+116 -1
View File
@@ -130,7 +130,7 @@ typedef struct SDMA_PKT_COPY_LINEAR_TAG {
static const size_t kMaxSize_ = 0x3fffe0;
} SDMA_PKT_COPY_LINEAR;
// linear sub-window
// linear sub-window (pre-GFX12)
typedef struct SDMA_PKT_COPY_LINEAR_RECT_TAG {
static const unsigned int pitch_bits = 19;
static const unsigned int slice_bits = 28;
@@ -253,6 +253,121 @@ typedef struct SDMA_PKT_COPY_LINEAR_RECT_TAG {
} SDMA_PKT_COPY_LINEAR_RECT;
// linear sub-window (GFX12)
typedef struct SDMA_PKT_COPY_LINEAR_RECT_TAG_GFX12 {
static const unsigned int pitch_bits = 16;
static const unsigned int slice_bits = 32;
static const unsigned int rect_xy_bits = 16;
static const unsigned int rect_z_bits = 14;
union {
struct {
unsigned int op : 8;
unsigned int sub_op : 8;
unsigned int reserved : 13;
unsigned int element : 3;
};
unsigned int DW_0_DATA;
} HEADER_UNION;
union {
struct {
unsigned int src_addr_31_0 : 32;
};
unsigned int DW_1_DATA;
} SRC_ADDR_LO_UNION;
union {
struct {
unsigned int src_addr_63_32 : 32;
};
unsigned int DW_2_DATA;
} SRC_ADDR_HI_UNION;
union {
struct {
unsigned int src_offset_x : 16;
unsigned int src_offset_y : 16;
};
unsigned int DW_3_DATA;
} SRC_PARAMETER_1_UNION;
union {
struct {
unsigned int src_offset_z : 14;
unsigned int reserved_1 : 2;
unsigned int src_pitch : pitch_bits;
};
unsigned int DW_4_DATA;
} SRC_PARAMETER_2_UNION;
union {
struct {
unsigned int src_slice_pitch : slice_bits;
};
unsigned int DW_5_DATA;
} SRC_PARAMETER_3_UNION;
union {
struct {
unsigned int dst_addr_31_0 : 32;
};
unsigned int DW_6_DATA;
} DST_ADDR_LO_UNION;
union {
struct {
unsigned int dst_addr_63_32 : 32;
};
unsigned int DW_7_DATA;
} DST_ADDR_HI_UNION;
union {
struct {
unsigned int dst_offset_x : 16;
unsigned int dst_offset_y : 16;
};
unsigned int DW_8_DATA;
} DST_PARAMETER_1_UNION;
union {
struct {
unsigned int dst_offset_z : 14;
unsigned int reserved_1 : 2;
unsigned int dst_pitch : pitch_bits;
};
unsigned int DW_9_DATA;
} DST_PARAMETER_2_UNION;
union {
struct {
unsigned int dst_slice_pitch : slice_bits;
};
unsigned int DW_10_DATA;
} DST_PARAMETER_3_UNION;
union {
struct {
unsigned int rect_x : rect_xy_bits;
unsigned int rect_y : rect_xy_bits;
};
unsigned int DW_11_DATA;
} RECT_PARAMETER_1_UNION;
union {
struct {
unsigned int rect_z : rect_z_bits;
unsigned int reserved_1 : 6;
unsigned int dst_cache_policy : 3;
unsigned int reserved_2 : 5;
unsigned int src_cache_policy : 3;
unsigned int reserved_3 : 1;
};
unsigned int DW_12_DATA;
} RECT_PARAMETER_2_UNION;
} SDMA_PKT_COPY_LINEAR_RECT_GFX12;
typedef struct SDMA_PKT_CONSTANT_FILL_TAG {
union {
struct {
projects/rocr-runtime/runtime/hsa-runtime/core/runtime/amd_blit_sdma.cpp
+59 -25
View File
@@ -553,7 +553,13 @@ BlitSdma<RingIndexTy, HwIndexMonotonic, SizeToCountOffset, useGCR>::SubmitCopyRe
if (range->z > 1 && (src->slice == 0 || dst->slice == 0))
throw AMD::hsa_exception(HSA_STATUS_ERROR_INVALID_ARGUMENT, "Copy rect slice needed.");
const uint max_pitch = 1 << SDMA_PKT_COPY_LINEAR_RECT::pitch_bits;
// GFX12 or later use a different packet format that is incompatible (fields changed in size and location).
const bool isGFX12Plus = (agent_->isa()->GetMajorVersion() >= 12);
// Common and GFX12 packet must match in size to use same code for vector/append.
static_assert(sizeof(SDMA_PKT_COPY_LINEAR_RECT) == sizeof(SDMA_PKT_COPY_LINEAR_RECT_GFX12));
const uint max_pitch = 1 << (isGFX12Plus ? SDMA_PKT_COPY_LINEAR_RECT_GFX12::pitch_bits : SDMA_PKT_COPY_LINEAR_RECT::pitch_bits);
std::vector<SDMA_PKT_COPY_LINEAR_RECT> pkts;
std::vector<uint64_t> bytes_moved;
@@ -838,12 +844,15 @@ void BlitSdma<RingIndexTy, HwIndexMonotonic, SizeToCountOffset, useGCR>::BuildCo
return __builtin_ctz(width | 16);
};
// GFX12 or later use a different packet format that is incompatible (fields changed in size and location).
const bool isGFX12Plus = (agent_->isa()->GetMajorVersion() >= 12);
// Limits in terms of element count
const uint32_t max_pitch = 1 << SDMA_PKT_COPY_LINEAR_RECT::pitch_bits;
const uint32_t max_slice = 1 << SDMA_PKT_COPY_LINEAR_RECT::slice_bits;
const uint32_t max_x = 1 << SDMA_PKT_COPY_LINEAR_RECT::rect_xy_bits;
const uint32_t max_y = 1 << SDMA_PKT_COPY_LINEAR_RECT::rect_xy_bits;
const uint32_t max_z = 1 << SDMA_PKT_COPY_LINEAR_RECT::rect_z_bits;
const uint32_t max_pitch = 1 << (isGFX12Plus ? SDMA_PKT_COPY_LINEAR_RECT_GFX12::pitch_bits : SDMA_PKT_COPY_LINEAR_RECT::pitch_bits);
const uint32_t max_slice = 1 << (isGFX12Plus ? SDMA_PKT_COPY_LINEAR_RECT_GFX12::slice_bits : SDMA_PKT_COPY_LINEAR_RECT::slice_bits);
const uint32_t max_x = 1 << (isGFX12Plus ? SDMA_PKT_COPY_LINEAR_RECT_GFX12::rect_xy_bits : SDMA_PKT_COPY_LINEAR_RECT::rect_xy_bits);
const uint32_t max_y = 1 << (isGFX12Plus ? SDMA_PKT_COPY_LINEAR_RECT_GFX12::rect_xy_bits : SDMA_PKT_COPY_LINEAR_RECT::rect_xy_bits);
const uint32_t max_z = 1 << (isGFX12Plus ? SDMA_PKT_COPY_LINEAR_RECT_GFX12::rect_z_bits : SDMA_PKT_COPY_LINEAR_RECT::rect_z_bits);
// Find maximum element that describes the pitch and slice.
// Pitch and slice must both be represented in units of elements. No element larger than this
@@ -918,27 +927,52 @@ void BlitSdma<RingIndexTy, HwIndexMonotonic, SizeToCountOffset, useGCR>::BuildCo
x += xcount << element;
SDMA_PKT_COPY_LINEAR_RECT* pkt =
(SDMA_PKT_COPY_LINEAR_RECT*)append(sizeof(SDMA_PKT_COPY_LINEAR_RECT));
*pkt = {};
pkt->HEADER_UNION.op = SDMA_OP_COPY;
pkt->HEADER_UNION.sub_op = SDMA_SUBOP_COPY_LINEAR_RECT;
pkt->HEADER_UNION.element = element;
pkt->SRC_ADDR_LO_UNION.src_addr_31_0 = sbase;
pkt->SRC_ADDR_HI_UNION.src_addr_63_32 = sbase >> 32;
pkt->SRC_PARAMETER_1_UNION.src_offset_x = soff;
pkt->SRC_PARAMETER_2_UNION.src_pitch = (src->pitch >> element) - 1;
pkt->SRC_PARAMETER_3_UNION.src_slice_pitch =
// GFX12 has a different packet format that is incompatible with pre-GFX12.
if (isGFX12Plus) {
SDMA_PKT_COPY_LINEAR_RECT_GFX12* pkt =
(SDMA_PKT_COPY_LINEAR_RECT_GFX12*)append(sizeof(SDMA_PKT_COPY_LINEAR_RECT));
*pkt = {};
pkt->HEADER_UNION.op = SDMA_OP_COPY;
pkt->HEADER_UNION.sub_op = SDMA_SUBOP_COPY_LINEAR_RECT;
pkt->HEADER_UNION.element = element;
pkt->SRC_ADDR_LO_UNION.src_addr_31_0 = sbase;
pkt->SRC_ADDR_HI_UNION.src_addr_63_32 = sbase >> 32;
pkt->SRC_PARAMETER_1_UNION.src_offset_x = soff;
pkt->SRC_PARAMETER_2_UNION.src_pitch = (src->pitch >> element) - 1;
pkt->SRC_PARAMETER_3_UNION.src_slice_pitch =
(range->z == 1) ? 0 : (src->slice >> element) - 1;
pkt->DST_ADDR_LO_UNION.dst_addr_31_0 = dbase;
pkt->DST_ADDR_HI_UNION.dst_addr_63_32 = dbase >> 32;
pkt->DST_PARAMETER_1_UNION.dst_offset_x = doff;
pkt->DST_PARAMETER_2_UNION.dst_pitch = (dst->pitch >> element) - 1;
pkt->DST_PARAMETER_3_UNION.dst_slice_pitch =
pkt->DST_ADDR_LO_UNION.dst_addr_31_0 = dbase;
pkt->DST_ADDR_HI_UNION.dst_addr_63_32 = dbase >> 32;
pkt->DST_PARAMETER_1_UNION.dst_offset_x = doff;
pkt->DST_PARAMETER_2_UNION.dst_pitch = (dst->pitch >> element) - 1;
pkt->DST_PARAMETER_3_UNION.dst_slice_pitch =
(range->z == 1) ? 0 : (dst->slice >> element) - 1;
pkt->RECT_PARAMETER_1_UNION.rect_x = xcount - 1;
pkt->RECT_PARAMETER_1_UNION.rect_y = Min(range->y - y, max_y) - 1;
pkt->RECT_PARAMETER_2_UNION.rect_z = Min(range->z - z, max_z) - 1;
pkt->RECT_PARAMETER_1_UNION.rect_x = xcount - 1;
pkt->RECT_PARAMETER_1_UNION.rect_y = Min(range->y - y, max_y) - 1;
pkt->RECT_PARAMETER_2_UNION.rect_z = Min(range->z - z, max_z) - 1;
} else { // Pre-GFX12, common packet used
SDMA_PKT_COPY_LINEAR_RECT* pkt =
(SDMA_PKT_COPY_LINEAR_RECT*)append(sizeof(SDMA_PKT_COPY_LINEAR_RECT));
*pkt = {};
pkt->HEADER_UNION.op = SDMA_OP_COPY;
pkt->HEADER_UNION.sub_op = SDMA_SUBOP_COPY_LINEAR_RECT;
pkt->HEADER_UNION.element = element;
pkt->SRC_ADDR_LO_UNION.src_addr_31_0 = sbase;
pkt->SRC_ADDR_HI_UNION.src_addr_63_32 = sbase >> 32;
pkt->SRC_PARAMETER_1_UNION.src_offset_x = soff;
pkt->SRC_PARAMETER_2_UNION.src_pitch = (src->pitch >> element) - 1;
pkt->SRC_PARAMETER_3_UNION.src_slice_pitch =
(range->z == 1) ? 0 : (src->slice >> element) - 1;
pkt->DST_ADDR_LO_UNION.dst_addr_31_0 = dbase;
pkt->DST_ADDR_HI_UNION.dst_addr_63_32 = dbase >> 32;
pkt->DST_PARAMETER_1_UNION.dst_offset_x = doff;
pkt->DST_PARAMETER_2_UNION.dst_pitch = (dst->pitch >> element) - 1;
pkt->DST_PARAMETER_3_UNION.dst_slice_pitch =
(range->z == 1) ? 0 : (dst->slice >> element) - 1;
pkt->RECT_PARAMETER_1_UNION.rect_x = xcount - 1;
pkt->RECT_PARAMETER_1_UNION.rect_y = Min(range->y - y, max_y) - 1;
pkt->RECT_PARAMETER_2_UNION.rect_z = Min(range->z - z, max_z) - 1;
}
}
}
}