AI/rocm-systems
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

add support for GPUs using wavefront size of 32 (#285)

Browse Source 
* add gfx1100 support

Add support for Radeon 7900 GPUs (RX and PRO), and 7800 PRO.

I was contemplating to add gfx1101 and gfx1102 GPUs as well, but those are the lower end models that are more unlikely to be used for compute intensive jobs. In addition, I do not have access to them to test the support.

* update WF_SIZe for different options

Radeon systems use a WarpSize of 32, unlike current Instinct systems,
which use a warp size of 64. For the device side, a gfx specific ifdef
is sufficient. For the host side, we need to query the device
properties.

* adjust functional tests to wf_size of 32

* update unit tests to handle wf_size of 32

* address reviewer comments

[ROCm/rocshmem commit: d0c2845031]
This commit is contained in:
Edgar Gabriel
2025-10-22 16:04:58 -05:00
committed by GitHub
parent b771a26916
commit d37af80d7e
19 changed files with 192 additions and 56 deletions
Download Patch File Download Diff File Expand all files Collapse all files
projects/rocshmem/tests/unit_tests/bitwise_gtest.cpp
+4 -10
View File
@@ -144,12 +144,6 @@ TEST_F(BitwiseTestFixture, verify_host_warp_matrix_init_1024_8) {
verify_zeroed_warp_matrix();
}
TEST_F(BitwiseTestFixture, verify_warp_size_64) {
setup_fixture({1, 1, 1}, {1, 1, 1});
ASSERT_EQ(WF_SIZE, 64);
}
/*****************************************************************************
************************** Activate Lane Helper******************************
*****************************************************************************/
@@ -1402,7 +1396,7 @@ TEST_F(BitwiseTestFixture, fetch_incr_kernel_4_1) {
for (size_t i = 0; i < _warp_matrix->rows(); i++) {
for (size_t j = 0; j < _warp_matrix->columns(); j++) {
auto *elem = _warp_matrix->access(i, j);
ASSERT_EQ(*elem % WF_SIZE, 0);
ASSERT_EQ(*elem % this->_wf_size, 0);
}
}
}
@@ -1426,7 +1420,7 @@ TEST_F(BitwiseTestFixture, fetch_incr_kernel_64_1) {
for (size_t i = 0; i < _warp_matrix->rows(); i++) {
for (size_t j = 0; j < _warp_matrix->columns(); j++) {
auto *elem = _warp_matrix->access(i, j);
ASSERT_EQ(*elem % WF_SIZE, 0);
ASSERT_EQ(*elem % this->_wf_size, 0);
}
}
}
@@ -1498,7 +1492,7 @@ TEST_F(BitwiseTestFixture, fetch_incr_kernel_1024_1024) {
for (size_t i = 0; i < _warp_matrix->rows(); i++) {
for (size_t j = 0; j < _warp_matrix->columns(); j++) {
auto *elem = _warp_matrix->access(i, j);
ASSERT_EQ(*elem % WF_SIZE, 0);
ASSERT_EQ(*elem % this->_wf_size, 0);
}
}
}
@@ -1522,7 +1516,7 @@ TEST_F(BitwiseTestFixture, fetch_incr_logical_1_kernel_1024_1024) {
for (size_t i = 0; i < _warp_matrix->rows(); i++) {
for (size_t j = 0; j < _warp_matrix->columns(); j++) {
auto *elem = _warp_matrix->access(i, j);
ASSERT_EQ(*elem % WF_SIZE, 1);
ASSERT_EQ(*elem % this->_wf_size, 1);
}
}
}
projects/rocshmem/tests/unit_tests/bitwise_gtest.hpp
+3
View File
@@ -28,6 +28,7 @@
#define HIP_ENABLE_PRINTF
#include "gtest/gtest.h"
#include "wf_size.hpp"
#include "../src/memory/hip_allocator.hpp"
#include "containers/matrix.hpp"
@@ -259,6 +260,7 @@ class BitwiseTestFixture : public ::testing::Test
{
_hip_block_dim = block_dim;
_hip_grid_dim = grid_dim;
_wf_size = get_wf_size();
assert(_device_methods == nullptr);
_hip_allocator.allocate(reinterpret_cast<void**>(&_device_methods),
@@ -341,6 +343,7 @@ class BitwiseTestFixture : public ::testing::Test
HIPAllocator _hip_allocator {};
WarpMatrix *_warp_matrix = nullptr;
BitwiseDeviceMethods *_device_methods = nullptr;
int _wf_size;
};
} // namespace rocshmem
projects/rocshmem/tests/unit_tests/free_list_gtest.cpp
+5 -5
View File
@@ -108,7 +108,7 @@ TYPED_TEST(FreeListTestFixture, push_host_pop_device) {
CHECK_HIP(hipMemset(results, 0, size_bytes));
is_empty = reinterpret_cast<bool*>(results + h_input.size());
const auto block_size = WF_SIZE;
const auto block_size = this->wf_size;
rocshmem::pop_all<<<1, block_size>>>(free_list, results, h_input.size());
CHECK_HIP(hipDeviceSynchronize());
@@ -140,7 +140,7 @@ TYPED_TEST(FreeListTestFixture, push_host_concurrent_pop_device) {
CHECK_HIP(hipMemset(results, 0, size_bytes));
is_empty = reinterpret_cast<bool*>(results + h_input.size());
const auto num_blocks = h_input.size();
const auto block_size = WF_SIZE;
const auto block_size = this->wf_size;
rocshmem::pop_all<<<num_blocks, block_size>>>(
free_list, results, h_input.size());
CHECK_HIP(hipDeviceSynchronize());
@@ -184,7 +184,7 @@ TYPED_TEST(FreeListTestFixture, push_host_pop_push_device) {
CHECK_HIP(hipMemset(results, 0, size_bytes));
d_input = reinterpret_cast<T*>(results + h_input.size());
is_empty = reinterpret_cast<bool*>(d_input + h_input.size());
const auto block_size = WF_SIZE;
const auto block_size = this->wf_size;
CHECK_HIP(hipMemcpy(d_input, h_input.data(), sizeof(T) * h_input.size(),
hipMemcpyHostToDevice));
@@ -223,7 +223,7 @@ TYPED_TEST(FreeListTestFixture, push_host_pop_concurrent_push_device) {
CHECK_HIP(hipMemset(results, 0, size_bytes));
d_input = reinterpret_cast<T*>(results + h_input.size());
const auto block_size = WF_SIZE;
const auto block_size = this->wf_size;
CHECK_HIP(hipMemcpy(d_input, h_input.data(), sizeof(T) * h_input.size(),
hipMemcpyHostToDevice));
@@ -277,7 +277,7 @@ TYPED_TEST(FreeListTestFixture, push_host_concurrent_pop_push_device) {
CHECK_HIP(hipMemcpy(d_input, h_input.data(), sizeof(T) * h_input.size(),
hipMemcpyHostToDevice));
const auto block_size = WF_SIZE;
const auto block_size = this->wf_size;
rocshmem::pop_all<FreeListType, T><<<1, block_size>>>(
free_list, nullptr, h_input.size());
CHECK_HIP(hipDeviceSynchronize());
projects/rocshmem/tests/unit_tests/free_list_gtest.hpp
+3
View File
@@ -31,6 +31,7 @@
#include "../src/containers/free_list_impl.hpp"
#include "gtest/gtest.h"
#include "../src/memory/hip_allocator.hpp"
#include "wf_size.hpp"
namespace rocshmem {
@@ -45,6 +46,7 @@ class FreeListTestFixture : public ::testing::Test {
protected:
void SetUp() override {
free_list->push_back_range(h_input.begin(), h_input.end());
wf_size = get_wf_size();
}
using T = ValueType;
@@ -52,6 +54,7 @@ class FreeListTestFixture : public ::testing::Test {
Allocator hip_allocator_ {};
const std::size_t num_elements{32};
std::vector<T> h_input{};
int wf_size;
FreeListProxy<Allocator, T> list_proxy{};
FreeList<T, Allocator>* free_list{};
projects/rocshmem/tests/unit_tests/wavefront_size_gtest.cpp
+28 -10
View File
@@ -28,16 +28,34 @@
using namespace rocshmem;
TEST_F(WavefrontSizeTestFixture, constant_matches_runtime) {
int device_count = 0;
hipDeviceProp_t prop;
CHECK_HIP(hipGetDeviceCount(&device_count));
ASSERT_GT(device_count, 0);
for (int i = 0; i < device_count; i++) {
CHECK_HIP(hipGetDeviceProperties(&prop, i));
ASSERT_EQ(WF_SIZE, prop.warpSize);
__global__ void check_wf_size(int wf_size_prop, int *ret) {
if (wf_size_prop == WF_SIZE) {
*ret = 0;
} else {
*ret = 1;
}
}
TEST_F(WavefrontSizeTestFixture, constant_matches_runtime) {
int device_count = 0;
hipDeviceProp_t prop;
int *ret;
CHECK_HIP(hipGetDeviceCount(&device_count));
ASSERT_GT(device_count, 0);
CHECK_HIP(hipHostMalloc(&ret, sizeof(int), 0));
for (int i = 0; i < device_count; i++) {
*ret = -1;
CHECK_HIP(hipSetDevice(i));
CHECK_HIP(hipGetDeviceProperties(&prop, i));
check_wf_size<<<1, 1>>>(prop.warpSize, ret);
CHECK_HIP(hipDeviceSynchronize());
ASSERT_EQ(*ret, 0);
}
CHECK_HIP(hipHostFree(ret));
}
projects/rocshmem/tests/unit_tests/wavefront_size_gtest.hpp
+10 -1
View File
@@ -26,10 +26,19 @@
#define ROCSHMEM_WAVEFRONT_SIZE_GTEST_HPP
#include "gtest/gtest.h"
#include "wf_size.hpp"
namespace rocshmem {
class WavefrontSizeTestFixture : public ::testing::Test { };
class WavefrontSizeTestFixture : public ::testing::Test {
public:
void SetUp() override {
wf_size = get_wf_size();
}
protected:
int wf_size;
};
} // namespace rocshmem
projects/rocshmem/tests/unit_tests/wf_size.hpp
+46
View File
@@ -0,0 +1,46 @@
/******************************************************************************
* Copyright (c) Advanced Micro Devices, Inc. All rights reserved.
*
* SPDX-License-Identifier: MIT
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to
* deal in the Software without restriction, including without limitation the
* rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
* sell copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in
* all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
* IN THE SOFTWARE.
*****************************************************************************/
#ifndef ROCSHMEM_WF_SIZE_HPP
#define ROCSHMEM_WF_SIZE_HPP
#include <hip/hip_runtime.h>
#include "mpi.h"
#define CHECK_HIP_MPI(cond) { \
if(cond != hipSuccess){ \
fprintf(stderr,"HIP error: %d line: %d\n", cond, __LINE__); \
MPI_Abort(MPI_COMM_WORLD, 1); \
} \
}
static int get_wf_size() {
int deviceId;
hipDeviceProp_t prop;
CHECK_HIP_MPI(hipGetDevice(&deviceId));
CHECK_HIP_MPI(hipGetDeviceProperties(&prop, deviceId));
return prop.warpSize;
}
#endif