Brandon Potter
@@ -0,0 +1,30 @@
|
||||
###############################################################################
|
||||
# Copyright (c) 2024 Advanced Micro Devices, Inc. All rights reserved.
|
||||
#
|
||||
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
# of this software and associated documentation files (the "Software"), to
|
||||
# deal in the Software without restriction, including without limitation the
|
||||
# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
|
||||
# sell copies of the Software, and to permit persons to whom the Software is
|
||||
# furnished to do so, subject to the following conditions:
|
||||
#
|
||||
# The above copyright notice and this permission notice shall be included in
|
||||
# all copies or substantial portions of the Software.
|
||||
#
|
||||
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
|
||||
# IN THE SOFTWARE.
|
||||
###############################################################################
|
||||
|
||||
###############################################################################
|
||||
# ADD ROCSHMEM TARGET FOR FILES IN CURRENT DIRECTORY
|
||||
###############################################################################
|
||||
target_sources(
|
||||
${PROJECT_NAME}
|
||||
PRIVATE
|
||||
host.cpp
|
||||
)
|
||||
@@ -0,0 +1,255 @@
|
||||
/******************************************************************************
|
||||
* Copyright (c) 2024 Advanced Micro Devices, Inc. All rights reserved.
|
||||
*
|
||||
* Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
* of this software and associated documentation files (the "Software"), to
|
||||
* deal in the Software without restriction, including without limitation the
|
||||
* rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
|
||||
* sell copies of the Software, and to permit persons to whom the Software is
|
||||
* furnished to do so, subject to the following conditions:
|
||||
*
|
||||
* The above copyright notice and this permission notice shall be included in
|
||||
* all copies or substantial portions of the Software.
|
||||
*
|
||||
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
|
||||
* IN THE SOFTWARE.
|
||||
*****************************************************************************/
|
||||
|
||||
#include "host.hpp"
|
||||
|
||||
#include <mpi.h>
|
||||
|
||||
#include "config.h" // NOLINT(build/include_subdir)
|
||||
#include "host_helpers.hpp"
|
||||
#include "../memory/window_info.hpp"
|
||||
#include "../util.hpp"
|
||||
|
||||
namespace rocshmem {
|
||||
|
||||
__host__ HostContextWindowInfo::HostContextWindowInfo(MPI_Comm comm_world,
|
||||
SymmetricHeap* heap) {
|
||||
window_info_ =
|
||||
new WindowInfo(comm_world, heap->get_local_heap_base(), heap->get_size());
|
||||
}
|
||||
|
||||
__host__ HostContextWindowInfo::~HostContextWindowInfo() {
|
||||
delete window_info_;
|
||||
}
|
||||
|
||||
WindowInfo* HostInterface::acquire_window_context() {
|
||||
auto index{find_avail_pool_entry()};
|
||||
|
||||
HostContextWindowInfo* acquired_win_info = host_window_context_pool_[index];
|
||||
|
||||
acquired_win_info->mark_unavail();
|
||||
|
||||
return acquired_win_info->get();
|
||||
}
|
||||
|
||||
__host__ void HostInterface::release_window_context(WindowInfo* window_info) {
|
||||
auto index{find_win_info_in_pool(window_info)};
|
||||
|
||||
host_window_context_pool_[index]->mark_avail();
|
||||
}
|
||||
|
||||
int HostInterface::find_avail_pool_entry() {
|
||||
for (int i{0}; i < max_num_ctxs_; i++) {
|
||||
if (host_window_context_pool_[i]->is_avail()) {
|
||||
return i;
|
||||
}
|
||||
}
|
||||
/* Entry should have been available; consider this as an error. */
|
||||
assert(false);
|
||||
return -1;
|
||||
}
|
||||
|
||||
int HostInterface::find_win_info_in_pool(WindowInfo* window_info) {
|
||||
for (int i{0}; i < max_num_ctxs_; i++) {
|
||||
if (host_window_context_pool_[i]->is_avail()) {
|
||||
continue;
|
||||
}
|
||||
if (window_info == host_window_context_pool_[i]->get()) {
|
||||
return i;
|
||||
}
|
||||
}
|
||||
/* Entry should have been present; consider this as an error. */
|
||||
assert(false);
|
||||
return -1;
|
||||
}
|
||||
|
||||
__host__ HostInterface::HostInterface(HdpPolicy* hdp_policy,
|
||||
MPI_Comm roc_shmem_comm,
|
||||
SymmetricHeap* heap) {
|
||||
/*
|
||||
* Duplicate a communicator from roc_shem's comm
|
||||
* world for the host interface
|
||||
*/
|
||||
MPI_Comm_dup(roc_shmem_comm, &host_comm_world_);
|
||||
MPI_Comm_rank(host_comm_world_, &my_pe_);
|
||||
MPI_Comm_rank(host_comm_world_, &num_pes_);
|
||||
|
||||
/*
|
||||
* Create an MPI window on the HDP so that it can be flushed
|
||||
* by remote PEs for host-facing functions
|
||||
*/
|
||||
hdp_policy_ = hdp_policy;
|
||||
|
||||
/*
|
||||
* Allocate and initialize pool of windows for contexts
|
||||
*/
|
||||
char* value{nullptr};
|
||||
if ((value = getenv("ROC_SHMEM_MAX_NUM_HOST_CONTEXTS"))) {
|
||||
max_num_ctxs_ = atoi(value);
|
||||
}
|
||||
|
||||
size_t pool_size = max_num_ctxs_ * sizeof(HostContextWindowInfo*);
|
||||
host_window_context_pool_ =
|
||||
reinterpret_cast<HostContextWindowInfo**>(malloc(pool_size));
|
||||
|
||||
for (int ctx_i = 0; ctx_i < max_num_ctxs_; ctx_i++) {
|
||||
host_window_context_pool_[ctx_i] =
|
||||
new HostContextWindowInfo(host_comm_world_, heap);
|
||||
}
|
||||
|
||||
#if !defined(USE_COHERENT_HEAP) && !defined(USE_SINGLE_NODE)
|
||||
// The single node implementation needs a different path since
|
||||
// the HDP flush pointers are allocated on the symmetric heap
|
||||
// and we need to wait for other initialization to happen before
|
||||
// calling `get_hdp_flush_ptr`.
|
||||
create_hdp_window();
|
||||
#endif // defined(USE_COHERENT_HEAP) && !defined(USE_SINGLE_NODE)
|
||||
}
|
||||
|
||||
#ifndef USE_COHERENT_HEAP
|
||||
__host__ void HostInterface::create_hdp_window() {
|
||||
MPI_Win_create(hdp_policy_->get_hdp_flush_ptr(),
|
||||
sizeof(unsigned int), /* size of window */
|
||||
sizeof(unsigned int), /* displacement */
|
||||
MPI_INFO_NULL, host_comm_world_, &hdp_win);
|
||||
|
||||
/*
|
||||
* Start a shared access epoch on windows of all ranks,
|
||||
* and let the library there is no need to check for
|
||||
* lock exclusivity during operations on this window
|
||||
* (MPI_MODE_NOCHECK).
|
||||
*/
|
||||
MPI_Win_lock_all(MPI_MODE_NOCHECK, hdp_win);
|
||||
}
|
||||
#endif // USE_COHERENT_HEAP
|
||||
|
||||
__host__ HostInterface::~HostInterface() {
|
||||
#ifndef USE_COHERENT_HEAP
|
||||
MPI_Win_unlock_all(hdp_win);
|
||||
|
||||
MPI_Win_free(&hdp_win);
|
||||
#endif // USE_COHERENT_HEAP
|
||||
|
||||
/* Detroy the pool of contexts */
|
||||
for (int ctx_i = 0; ctx_i < max_num_ctxs_; ctx_i++) {
|
||||
delete host_window_context_pool_[ctx_i];
|
||||
}
|
||||
|
||||
free(host_window_context_pool_);
|
||||
|
||||
MPI_Comm_free(&host_comm_world_);
|
||||
}
|
||||
|
||||
__host__ void HostInterface::putmem_nbi(void* dest, const void* source,
|
||||
size_t nelems, int pe,
|
||||
WindowInfo* window_info) {
|
||||
initiate_put(dest, source, nelems, pe, window_info);
|
||||
}
|
||||
|
||||
__host__ void HostInterface::getmem_nbi(void* dest, const void* source,
|
||||
size_t nelems, int pe,
|
||||
WindowInfo* window_info) {
|
||||
initiate_get(dest, source, nelems, pe, window_info);
|
||||
}
|
||||
|
||||
__host__ void HostInterface::putmem(void* dest, const void* source,
|
||||
size_t nelems, int pe,
|
||||
WindowInfo* window_info) {
|
||||
initiate_put(dest, source, nelems, pe, window_info);
|
||||
|
||||
MPI_Win_flush_local(pe, window_info->get_win());
|
||||
}
|
||||
|
||||
__host__ void HostInterface::getmem(void* dest, const void* source,
|
||||
size_t nelems, int pe,
|
||||
WindowInfo* window_info) {
|
||||
initiate_get(dest, source, nelems, pe, window_info);
|
||||
|
||||
MPI_Win_flush_local(pe, window_info->get_win());
|
||||
|
||||
/*
|
||||
* Flush local HDP to ensure that the NIC's write
|
||||
* of the fetched data is visible in device memory
|
||||
*/
|
||||
hdp_policy_->hdp_flush();
|
||||
}
|
||||
|
||||
__host__ void HostInterface::fence(WindowInfo* window_info) {
|
||||
complete_all(window_info->get_win());
|
||||
|
||||
/*
|
||||
* Flush my HDP and the HDPs of remote GPUs.
|
||||
* The HDP is a write-combining (WC) write-through
|
||||
* cache. But, even after the WC buffer is full and
|
||||
* the data is passed to the Data Fabric (DF), DF
|
||||
* can still reorder the writes. A flush ensures
|
||||
* that writes after the flush are written only
|
||||
* after those before the flush.
|
||||
*/
|
||||
hdp_policy_->hdp_flush();
|
||||
flush_remote_hdps();
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
__host__ void HostInterface::quiet(WindowInfo* window_info) {
|
||||
complete_all(window_info->get_win());
|
||||
|
||||
/* Same explanation as in fence */
|
||||
hdp_policy_->hdp_flush();
|
||||
flush_remote_hdps();
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
__host__ void HostInterface::sync_all(WindowInfo* window_info) {
|
||||
MPI_Win_sync(window_info->get_win());
|
||||
|
||||
hdp_policy_->hdp_flush();
|
||||
/*
|
||||
* No need to flush remote
|
||||
* HDPs here since all PEs are
|
||||
* participating.
|
||||
*/
|
||||
|
||||
MPI_Barrier(host_comm_world_);
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
__host__ void HostInterface::barrier_all(WindowInfo* window_info) {
|
||||
complete_all(window_info->get_win());
|
||||
|
||||
/*
|
||||
* Flush my HDP cache so remote NICs will
|
||||
* see the latest values in device memory
|
||||
*/
|
||||
hdp_policy_->hdp_flush();
|
||||
|
||||
MPI_Barrier(host_comm_world_);
|
||||
}
|
||||
|
||||
__host__ void HostInterface::barrier_for_sync() {
|
||||
MPI_Barrier(host_comm_world_);
|
||||
}
|
||||
|
||||
} // namespace rocshmem
|
||||
@@ -0,0 +1,406 @@
|
||||
/******************************************************************************
|
||||
* Copyright (c) 2024 Advanced Micro Devices, Inc. All rights reserved.
|
||||
*
|
||||
* Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
* of this software and associated documentation files (the "Software"), to
|
||||
* deal in the Software without restriction, including without limitation the
|
||||
* rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
|
||||
* sell copies of the Software, and to permit persons to whom the Software is
|
||||
* furnished to do so, subject to the following conditions:
|
||||
*
|
||||
* The above copyright notice and this permission notice shall be included in
|
||||
* all copies or substantial portions of the Software.
|
||||
*
|
||||
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
|
||||
* IN THE SOFTWARE.
|
||||
*****************************************************************************/
|
||||
|
||||
#ifndef LIBRARY_SRC_HOST_HOST_HPP_
|
||||
#define LIBRARY_SRC_HOST_HOST_HPP_
|
||||
|
||||
/**
|
||||
* @file host.hpp
|
||||
* Defines the HostInterface class.
|
||||
*
|
||||
* The file contains the HostInterface class that defines all the
|
||||
* host-facing functions that will be used by all host contexts of
|
||||
* any backend type.
|
||||
*/
|
||||
|
||||
#include <mpi.h>
|
||||
|
||||
#include <map>
|
||||
|
||||
#include "roc_shmem/roc_shmem.hpp"
|
||||
#include "../hdp_policy.hpp"
|
||||
#include "../memory/symmetric_heap.hpp"
|
||||
#include "../memory/window_info.hpp"
|
||||
|
||||
namespace rocshmem {
|
||||
|
||||
class HostContextWindowInfo {
|
||||
public:
|
||||
/**
|
||||
* @brief Constructor with default members
|
||||
*/
|
||||
HostContextWindowInfo() = default;
|
||||
|
||||
/**
|
||||
* @brief Constructor with initialized members
|
||||
*
|
||||
* @param[in] team pointer used to track team info
|
||||
* @param[in] team_info information about participating PEs
|
||||
*/
|
||||
HostContextWindowInfo(MPI_Comm comm_world, SymmetricHeap* heap);
|
||||
|
||||
/**
|
||||
* @brief Destructor
|
||||
*/
|
||||
__host__ ~HostContextWindowInfo();
|
||||
|
||||
/**
|
||||
* @brief Retrieve a pointer to the internal WindowInfo
|
||||
*
|
||||
* @return WindowInfo pointer
|
||||
*/
|
||||
WindowInfo* get() { return window_info_; }
|
||||
|
||||
/**
|
||||
* @brief Mark the window info as avaialable (not allocated)
|
||||
*/
|
||||
void mark_avail() { avail_ = true; }
|
||||
|
||||
/**
|
||||
* @brief Mark the window info as unavaialble (allocated)
|
||||
*/
|
||||
void mark_unavail() { avail_ = false; }
|
||||
|
||||
/**
|
||||
* @brief Check if the window info has been allocated
|
||||
*/
|
||||
bool is_avail() { return avail_; }
|
||||
|
||||
private:
|
||||
/**
|
||||
* @brief Flag to state whether or not this window is available to be assigned
|
||||
* to a Host Context
|
||||
*/
|
||||
bool avail_{true};
|
||||
|
||||
/**
|
||||
* @brief Pointer to the WindowInfo object that manages the MPI Window for
|
||||
* this context
|
||||
*/
|
||||
WindowInfo* window_info_{nullptr};
|
||||
};
|
||||
|
||||
class HostInterface {
|
||||
public:
|
||||
/**
|
||||
* @brief Primary constructor
|
||||
*/
|
||||
__host__ HostInterface(HdpPolicy* hdp_policy, MPI_Comm roc_shmem_comm,
|
||||
SymmetricHeap* heap);
|
||||
|
||||
/**
|
||||
* @brief Destructor
|
||||
*/
|
||||
__host__ ~HostInterface();
|
||||
|
||||
/**
|
||||
* @brief Accessor for copy of comm world
|
||||
*
|
||||
* @return MPI_Comm containing host comm world
|
||||
*/
|
||||
MPI_Comm get_comm_world() { return host_comm_world_; }
|
||||
|
||||
/**
|
||||
* @brief Get a window context from the pool
|
||||
*
|
||||
* @return Pointer to the WindowInfo in the allocated one from the pool
|
||||
*/
|
||||
WindowInfo* acquire_window_context();
|
||||
|
||||
/**
|
||||
* @brief Return a window context back to the pool
|
||||
*/
|
||||
void release_window_context(WindowInfo* window_info);
|
||||
|
||||
/**************************************************************************
|
||||
***************************** HOST FUNCTIONS *****************************
|
||||
*************************************************************************/
|
||||
template <typename T>
|
||||
__host__ void p(T* dest, T value, int pe, WindowInfo* window_info);
|
||||
|
||||
template <typename T>
|
||||
__host__ T g(const T* source, int pe, WindowInfo* window_info);
|
||||
|
||||
template <typename T>
|
||||
__host__ void put(T* dest, const T* source, size_t nelems, int pe,
|
||||
WindowInfo* window_info);
|
||||
|
||||
template <typename T>
|
||||
__host__ void get(T* dest, const T* source, size_t nelems, int pe,
|
||||
WindowInfo* window_info);
|
||||
|
||||
template <typename T>
|
||||
__host__ void put_nbi(T* dest, const T* source, size_t nelems, int pe,
|
||||
WindowInfo* window_info);
|
||||
|
||||
template <typename T>
|
||||
__host__ void get_nbi(T* dest, const T* source, size_t nelems, int pe,
|
||||
WindowInfo* window_info);
|
||||
|
||||
__host__ void putmem(void* dest, const void* source, size_t nelems, int pe,
|
||||
WindowInfo* window_info);
|
||||
|
||||
__host__ void getmem(void* dest, const void* source, size_t nelems, int pe,
|
||||
WindowInfo* window_info);
|
||||
|
||||
__host__ void putmem_nbi(void* dest, const void* source, size_t nelems,
|
||||
int pe, WindowInfo* window_info);
|
||||
|
||||
__host__ void getmem_nbi(void* dest, const void* source, size_t size, int pe,
|
||||
WindowInfo* window_info);
|
||||
|
||||
template <typename T>
|
||||
__host__ void amo_add(void* dst, T value, int pe, WindowInfo* window_info);
|
||||
|
||||
template <typename T>
|
||||
__host__ void amo_cas(void* dst, T value, T cond, int pe,
|
||||
WindowInfo* window_info);
|
||||
|
||||
template <typename T>
|
||||
__host__ T amo_fetch_add(void* dst, T value, int pe, WindowInfo* window_info);
|
||||
|
||||
template <typename T>
|
||||
__host__ T amo_fetch_cas(void* dst, T value, T cond, int pe,
|
||||
WindowInfo* window_info);
|
||||
|
||||
__host__ void fence(WindowInfo* window_info);
|
||||
|
||||
__host__ void quiet(WindowInfo* window_info);
|
||||
|
||||
__host__ void barrier_all(WindowInfo* window_info);
|
||||
|
||||
__host__ void barrier_for_sync();
|
||||
|
||||
__host__ void sync_all(WindowInfo* window_info);
|
||||
|
||||
template <typename T>
|
||||
__host__ void broadcast(T* dest, const T* source, int nelems, int pe_root,
|
||||
int pe_start, int log_pe_stride, int pe_size,
|
||||
long* p_sync); // NOLINT(runtime/int)
|
||||
|
||||
template <typename T>
|
||||
__host__ void broadcast(roc_shmem_team_t team, T* dest, const T* source,
|
||||
int nelems, int pe_root);
|
||||
|
||||
template <typename T, ROC_SHMEM_OP Op>
|
||||
__host__ void to_all(T* dest, const T* source, int nreduce, int pe_start,
|
||||
int log_pe_stride, int pe_size, T* p_wrk,
|
||||
long* p_sync); // NOLINT(runtime/int)
|
||||
|
||||
template <typename T, ROC_SHMEM_OP Op>
|
||||
__host__ void to_all(roc_shmem_team_t team, T* dest, const T* source,
|
||||
int nreduce);
|
||||
|
||||
template <typename T>
|
||||
__host__ void wait_until(T* ptr, roc_shmem_cmps cmp, T val,
|
||||
WindowInfo* window_info);
|
||||
|
||||
template <typename T>
|
||||
__host__ void wait_until_all(T* ptr, size_t nelems, const int* status,
|
||||
roc_shmem_cmps cmp, T val,
|
||||
WindowInfo* window_info);
|
||||
|
||||
template <typename T>
|
||||
__host__ size_t wait_until_any(T* ptr, size_t nelems, const int* status,
|
||||
roc_shmem_cmps cmp, T val,
|
||||
WindowInfo* window_info);
|
||||
|
||||
template <typename T>
|
||||
__host__ size_t wait_until_some(T* ptr, size_t nelems, size_t* indices,
|
||||
const int* status, roc_shmem_cmps cmp, T val,
|
||||
WindowInfo* window_info);
|
||||
|
||||
template <typename T>
|
||||
__host__ void wait_until_all_vector(T* ptr, size_t nelems, const int* status,
|
||||
roc_shmem_cmps cmp, T* vals,
|
||||
WindowInfo* window_info);
|
||||
|
||||
template <typename T>
|
||||
__host__ size_t wait_until_any_vector(T* ptr, size_t nelems,
|
||||
const int* status, roc_shmem_cmps cmp,
|
||||
T* vals, WindowInfo* window_info);
|
||||
|
||||
template <typename T>
|
||||
__host__ size_t wait_until_some_vector(T* ptr, size_t nelems, size_t* indices,
|
||||
const int* status, roc_shmem_cmps cmp,
|
||||
T* vals, WindowInfo* window_info);
|
||||
|
||||
template <typename T>
|
||||
__host__ int test(T* ptr, roc_shmem_cmps cmp, T val, WindowInfo* window_info);
|
||||
|
||||
#ifndef USE_COHERENT_HEAP
|
||||
__host__ void create_hdp_window();
|
||||
#endif // USE_COHERENT_HEAP
|
||||
|
||||
private:
|
||||
/**************************************************************************
|
||||
**************************** INTERNAL METHODS ****************************
|
||||
*************************************************************************/
|
||||
__host__ void flush_remote_hdps() {
|
||||
#ifndef USE_COHERENT_HEAP
|
||||
unsigned flush_val{HdpPolicy::HDP_FLUSH_VAL};
|
||||
for (size_t i{0}; i < num_pes_; i++) {
|
||||
if (i == my_pe_) {
|
||||
continue;
|
||||
}
|
||||
MPI_Put(&flush_val, 1, MPI_UNSIGNED, i, 0, 1, MPI_UNSIGNED, hdp_win);
|
||||
}
|
||||
MPI_Win_flush_all(hdp_win);
|
||||
#endif // USE_COHERENT_HEAP
|
||||
}
|
||||
|
||||
__host__ void flush_remote_hdp(int pe) {
|
||||
#ifndef USE_COHERENT_HEAP
|
||||
unsigned flush_val{HdpPolicy::HDP_FLUSH_VAL};
|
||||
MPI_Put(&flush_val, 1, MPI_UNSIGNED, pe, 0, 1, MPI_UNSIGNED, hdp_win);
|
||||
MPI_Win_flush(pe, hdp_win);
|
||||
#endif // USE_COHERENT_HEAP
|
||||
}
|
||||
|
||||
__host__ void initiate_put(void* dest, const void* source, size_t nelems,
|
||||
int pe, WindowInfo* window_info);
|
||||
|
||||
__host__ void initiate_get(void* dest, const void* source, size_t nelems,
|
||||
int pe, WindowInfo* window_info);
|
||||
|
||||
__host__ void complete_all(MPI_Win win);
|
||||
|
||||
__host__ MPI_Aint compute_offset(const void* dest, void* win_start,
|
||||
void* win_end);
|
||||
|
||||
__host__ MPI_Comm get_mpi_comm(int pe_start, int log_pe_stride, int pe_size);
|
||||
|
||||
__host__ MPI_Op get_mpi_op(ROC_SHMEM_OP Op);
|
||||
|
||||
template <typename T>
|
||||
__host__ MPI_Datatype get_mpi_type();
|
||||
|
||||
template <typename T>
|
||||
__host__ int compare(roc_shmem_cmps cmp, T input_val, T target_val);
|
||||
|
||||
template <typename T>
|
||||
__host__ int test_and_compare(MPI_Aint offset, MPI_Datatype mpi_type,
|
||||
roc_shmem_cmps cmp, T val, MPI_Win win);
|
||||
|
||||
template <typename T, ROC_SHMEM_OP Op>
|
||||
__host__ void to_all_internal(MPI_Comm mpi_comm, T* dest, const T* source,
|
||||
int nreduce);
|
||||
|
||||
template <typename T>
|
||||
__host__ void broadcast_internal(MPI_Comm mpi_comm, T* dest, const T* source,
|
||||
int nelems, int pe_root);
|
||||
|
||||
/**************************************************************************
|
||||
**************************** INTERNAL MEMBERS ****************************
|
||||
*************************************************************************/
|
||||
/**
|
||||
* @brief Duplicate to the Backend's hdp policy pointer
|
||||
*/
|
||||
HdpPolicy* hdp_policy_{nullptr};
|
||||
|
||||
/**
|
||||
* @brief Global MPI communicator for those host API
|
||||
*/
|
||||
MPI_Comm host_comm_world_{};
|
||||
|
||||
/**
|
||||
* @brief Duplicate of this processing element's id within global rank
|
||||
*/
|
||||
int my_pe_{-1};
|
||||
|
||||
/**
|
||||
* @brief Duplicate of global number of processing elements
|
||||
*/
|
||||
int num_pes_{0};
|
||||
|
||||
#ifndef USE_COHERENT_HEAP
|
||||
/**
|
||||
* @brief MPI window for hdp flushing
|
||||
*/
|
||||
MPI_Win hdp_win;
|
||||
#endif // USE_COHERENT_HEAP
|
||||
|
||||
/**
|
||||
* @brief Max number of contexts for the application
|
||||
*/
|
||||
int max_num_ctxs_{40};
|
||||
|
||||
/**
|
||||
* @brief Pool of HostContexWindowInfos
|
||||
*/
|
||||
HostContextWindowInfo** host_window_context_pool_{nullptr};
|
||||
|
||||
int find_win_info_in_pool(WindowInfo* window_info);
|
||||
|
||||
int find_avail_pool_entry();
|
||||
|
||||
/*
|
||||
* @brief Used by comm_map map for active sets.
|
||||
*
|
||||
* This data structure that stores the parameters defining the active
|
||||
* set of PEs in a collective. This struct also serves as a key
|
||||
* into the comm_map map.
|
||||
*/
|
||||
class ActiveSetKey {
|
||||
public:
|
||||
/**
|
||||
* @brief Primary constructor
|
||||
*/
|
||||
ActiveSetKey(int pe_start, int log_pe_stride, int pe_size)
|
||||
: pe_start_(pe_start),
|
||||
log_pe_stride_(log_pe_stride),
|
||||
pe_size_(pe_size) {}
|
||||
|
||||
bool operator<(const ActiveSetKey& key) const {
|
||||
return pe_start_ < key.pe_start_ ||
|
||||
(pe_start_ == key.pe_start_ &&
|
||||
log_pe_stride_ < key.log_pe_stride_) ||
|
||||
(pe_start_ == key.pe_start_ &&
|
||||
log_pe_stride_ == key.log_pe_stride_ && pe_size_ < key.pe_size_);
|
||||
}
|
||||
|
||||
private:
|
||||
/**
|
||||
* @brief Records start location in (logical) active set bitmask
|
||||
*/
|
||||
int pe_start_{-1};
|
||||
|
||||
/**
|
||||
* @brief Records stride in (logical) active set bitmask
|
||||
*/
|
||||
int log_pe_stride_{-1};
|
||||
|
||||
/**
|
||||
* @brief Records (logical) active set bitmask size
|
||||
*/
|
||||
int pe_size_{-1};
|
||||
};
|
||||
|
||||
/*
|
||||
* @brief Map of active set descriptors to MPI communicators
|
||||
*/
|
||||
std::map<ActiveSetKey, MPI_Comm> comm_map{};
|
||||
};
|
||||
|
||||
} // namespace rocshmem
|
||||
|
||||
#endif // LIBRARY_SRC_HOST_HOST_HPP_
|
||||
@@ -0,0 +1,92 @@
|
||||
/******************************************************************************
|
||||
* Copyright (c) 2024 Advanced Micro Devices, Inc. All rights reserved.
|
||||
*
|
||||
* Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
* of this software and associated documentation files (the "Software"), to
|
||||
* deal in the Software without restriction, including without limitation the
|
||||
* rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
|
||||
* sell copies of the Software, and to permit persons to whom the Software is
|
||||
* furnished to do so, subject to the following conditions:
|
||||
*
|
||||
* The above copyright notice and this permission notice shall be included in
|
||||
* all copies or substantial portions of the Software.
|
||||
*
|
||||
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
|
||||
* IN THE SOFTWARE.
|
||||
*****************************************************************************/
|
||||
|
||||
#ifndef LIBRARY_SRC_HOST_HOST_HELPERS_HPP_
|
||||
#define LIBRARY_SRC_HOST_HOST_HELPERS_HPP_
|
||||
|
||||
#include "host.hpp"
|
||||
#include "../memory/window_info.hpp"
|
||||
|
||||
namespace rocshmem {
|
||||
|
||||
__host__ inline MPI_Aint HostInterface::compute_offset(
|
||||
const void* dest, void* win_start, [[maybe_unused]] void* win_end) {
|
||||
assert((reinterpret_cast<char*>(const_cast<void*>(dest)) >=
|
||||
reinterpret_cast<char*>(win_start)) &&
|
||||
(reinterpret_cast<char*>(const_cast<void*>(dest)) <
|
||||
reinterpret_cast<char*>(win_end)));
|
||||
|
||||
MPI_Aint dest_disp{};
|
||||
MPI_Aint start_disp{};
|
||||
|
||||
MPI_Get_address(dest, &dest_disp);
|
||||
MPI_Get_address(win_start, &start_disp);
|
||||
|
||||
return MPI_Aint_diff(dest_disp, start_disp);
|
||||
}
|
||||
|
||||
__host__ inline void HostInterface::complete_all(MPI_Win win) {
|
||||
MPI_Win_flush_all(win); /* RMA operations */
|
||||
MPI_Win_sync(win); /* memory stores */
|
||||
}
|
||||
|
||||
__host__ inline void HostInterface::initiate_put(void* dest, const void* source,
|
||||
size_t nelems, int pe,
|
||||
WindowInfo* window_info) {
|
||||
MPI_Win win{window_info->get_win()};
|
||||
void* win_start{window_info->get_start()};
|
||||
void* win_end{window_info->get_end()};
|
||||
|
||||
/* Calculate offset of remote dest from base address of window */
|
||||
MPI_Aint offset{compute_offset(dest, win_start, win_end)};
|
||||
|
||||
/*
|
||||
* Current semantics of our API restrict the buffers
|
||||
* passed in to be on the symmetric heap only. So,
|
||||
* flush the HDP since the GPU may have written the
|
||||
* latest value to the source buffer and we want the
|
||||
* NIC to DMA read the latest value instead of the
|
||||
* value that may have been cached in the HDP.
|
||||
*/
|
||||
hdp_policy_->hdp_flush();
|
||||
|
||||
/* Offload remote write operation to MPI */
|
||||
MPI_Put(source, nelems, MPI_CHAR, pe, offset, nelems, MPI_CHAR, win);
|
||||
}
|
||||
|
||||
__host__ inline void HostInterface::initiate_get(void* dest, const void* source,
|
||||
size_t nelems, int pe,
|
||||
WindowInfo* window_info) {
|
||||
MPI_Win win{window_info->get_win()};
|
||||
void* win_start{window_info->get_start()};
|
||||
void* win_end{window_info->get_end()};
|
||||
|
||||
/* Calculate offset of remote source from base address of window */
|
||||
MPI_Aint offset = compute_offset(source, win_start, win_end);
|
||||
|
||||
/* Offload remote fetch operation to MPI */
|
||||
MPI_Get(dest, nelems, MPI_CHAR, pe, offset, nelems, MPI_CHAR, win);
|
||||
}
|
||||
|
||||
} // namespace rocshmem
|
||||
|
||||
#endif // LIBRARY_SRC_HOST_HOST_HELPERS_HPP_
|
||||
@@ -0,0 +1,645 @@
|
||||
/******************************************************************************
|
||||
* Copyright (c) 2024 Advanced Micro Devices, Inc. All rights reserved.
|
||||
*
|
||||
* Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
* of this software and associated documentation files (the "Software"), to
|
||||
* deal in the Software without restriction, including without limitation the
|
||||
* rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
|
||||
* sell copies of the Software, and to permit persons to whom the Software is
|
||||
* furnished to do so, subject to the following conditions:
|
||||
*
|
||||
* The above copyright notice and this permission notice shall be included in
|
||||
* all copies or substantial portions of the Software.
|
||||
*
|
||||
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
|
||||
* IN THE SOFTWARE.
|
||||
*****************************************************************************/
|
||||
#ifndef LIBRARY_SRC_HOST_HOST_TEMPLATES_HPP_
|
||||
#define LIBRARY_SRC_HOST_HOST_TEMPLATES_HPP_
|
||||
|
||||
#include <utility>
|
||||
|
||||
#include "config.h" // NOLINT(build/include_subdir)
|
||||
#include "host_helpers.hpp"
|
||||
#include "../memory/window_info.hpp"
|
||||
#include "../team.hpp"
|
||||
|
||||
namespace rocshmem {
|
||||
|
||||
template <typename T>
|
||||
__host__ void HostInterface::p(T* dest, T value, int pe,
|
||||
WindowInfo* window_info) {
|
||||
DPRINTF("Function: host_p\n");
|
||||
putmem(dest, &value, sizeof(T), pe, window_info);
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
__host__ void HostInterface::put(T* dest, const T* source, size_t nelems,
|
||||
int pe, WindowInfo* window_info) {
|
||||
DPRINTF("Function: host_put\n");
|
||||
putmem(dest, source, sizeof(T) * nelems, pe, window_info);
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
__host__ void HostInterface::put_nbi(T* dest, const T* source, size_t nelems,
|
||||
int pe, WindowInfo* window_info) {
|
||||
DPRINTF("Function: host_put_nbi\n");
|
||||
putmem_nbi(dest, source, sizeof(T) * nelems, pe, window_info);
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
__host__ T HostInterface::g(const T* source, int pe, WindowInfo* window_info) {
|
||||
DPRINTF("Function: host_g\n");
|
||||
|
||||
T ret{};
|
||||
|
||||
/*
|
||||
* We don't call getmem directly here
|
||||
* since it flushes the local HDP. We
|
||||
* don't need the flush because the
|
||||
* destination buffer is on the CPU.
|
||||
*/
|
||||
getmem_nbi(&ret, source, sizeof(T), pe, window_info);
|
||||
|
||||
MPI_Win_flush_local(pe, window_info->get_win());
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
__host__ void HostInterface::get(T* dest, const T* source, size_t nelems,
|
||||
int pe, WindowInfo* window_info) {
|
||||
DPRINTF("Function: host_get\n");
|
||||
getmem(dest, source, sizeof(T) * nelems, pe, window_info);
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
__host__ void HostInterface::get_nbi(T* dest, const T* source, size_t nelems,
|
||||
int pe, WindowInfo* window_info) {
|
||||
DPRINTF("Function: host_get_nbi\n");
|
||||
getmem_nbi(dest, source, sizeof(T) * nelems, pe, window_info);
|
||||
}
|
||||
|
||||
__host__ MPI_Comm HostInterface::get_mpi_comm(int pe_start, int log_pe_stride,
|
||||
int pe_size) {
|
||||
MPI_Comm active_set_comm{};
|
||||
|
||||
/*
|
||||
* First, check to see if the active set is the same as COMM_WORLD
|
||||
*/
|
||||
int comm_world_size{-1};
|
||||
MPI_Comm_size(host_comm_world_, &comm_world_size);
|
||||
|
||||
if (pe_start == 0 && log_pe_stride == 0 && pe_size == comm_world_size) {
|
||||
/*
|
||||
* Use the host interface's copy of MPI_COMM_WORLD
|
||||
* TODO: replace with a per-context copy of MPI_COMM_WORLD when we
|
||||
* have multiple contexts
|
||||
*/
|
||||
active_set_comm = host_comm_world_;
|
||||
return active_set_comm;
|
||||
}
|
||||
|
||||
/*
|
||||
* Then, check to see if we had already created a communicator for
|
||||
* this active set
|
||||
*/
|
||||
ActiveSetKey key(pe_start, log_pe_stride, pe_size);
|
||||
|
||||
auto it{comm_map.find(key)};
|
||||
if (it != comm_map.end()) {
|
||||
DPRINTF("Using cached communicator\n");
|
||||
return it->second;
|
||||
}
|
||||
|
||||
/*
|
||||
* If there is not one cached, create a new one (expensive)
|
||||
*/
|
||||
int active_set_ranks[pe_size]; // NOLINT
|
||||
int stride{1 << log_pe_stride};
|
||||
active_set_ranks[0] = pe_start;
|
||||
|
||||
for (int i{1}; i < pe_size; i++) {
|
||||
active_set_ranks[i] = active_set_ranks[i - 1] + stride;
|
||||
}
|
||||
|
||||
MPI_Group comm_world_group{};
|
||||
MPI_Group active_set_group{};
|
||||
|
||||
MPI_Comm_group(host_comm_world_, &comm_world_group);
|
||||
|
||||
MPI_Group_incl(comm_world_group, pe_size, active_set_ranks,
|
||||
&active_set_group);
|
||||
|
||||
MPI_Comm_create_group(host_comm_world_, active_set_group, 0,
|
||||
&active_set_comm);
|
||||
|
||||
/*
|
||||
* Cache the new communicator
|
||||
*/
|
||||
DPRINTF("Created a new communicator. Now caching it\n");
|
||||
comm_map.insert(std::pair<ActiveSetKey, MPI_Comm>(key, active_set_comm));
|
||||
|
||||
return active_set_comm;
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
__host__ void HostInterface::broadcast_internal(MPI_Comm mpi_comm, T* dest,
|
||||
const T* source, int nelems,
|
||||
int pe_root) {
|
||||
DPRINTF("Function: host_broadcast_internal\n");
|
||||
|
||||
/*
|
||||
* Choose the right pointer for my buffer depending
|
||||
* on whether or not I am the root.
|
||||
*/
|
||||
int active_set_rank{-1};
|
||||
void* buffer{nullptr};
|
||||
MPI_Comm_rank(mpi_comm, &active_set_rank);
|
||||
if (pe_root == active_set_rank) {
|
||||
buffer = const_cast<T*>(source);
|
||||
} else {
|
||||
buffer = const_cast<T*>(dest);
|
||||
}
|
||||
|
||||
/*
|
||||
* Flush my HDP so that the NIC does not read stale values
|
||||
*/
|
||||
hdp_policy_->hdp_flush();
|
||||
|
||||
/*
|
||||
* Offload the broadcast to MPI
|
||||
*/
|
||||
MPI_Bcast(buffer, nelems * sizeof(T), MPI_CHAR, pe_root, mpi_comm);
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
__host__ void HostInterface::broadcast(T* dest, const T* source, int nelems,
|
||||
int pe_root, int pe_start,
|
||||
int log_pe_stride, int pe_size,
|
||||
[[maybe_unused]] long* p_sync) {
|
||||
DPRINTF("Function: host_broadcast\n");
|
||||
|
||||
/*
|
||||
* Get an MPI communicator for active set of PEs
|
||||
* Note: pe_root is w.r.t the active set, hence
|
||||
* the MPI communicator contains the root as well.
|
||||
*/
|
||||
MPI_Comm mpi_comm{get_mpi_comm(pe_start, log_pe_stride, pe_size)};
|
||||
|
||||
broadcast_internal<T>(mpi_comm, dest, source, nelems, pe_root);
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
__host__ void HostInterface::broadcast(roc_shmem_team_t team, T* dest,
|
||||
const T* source, int nelems,
|
||||
int pe_root) {
|
||||
DPRINTF("Function: Team-based host_broadcast\n");
|
||||
|
||||
/*
|
||||
* Get the MPI communicator of this team
|
||||
*/
|
||||
Team* team_obj{get_internal_team(team)};
|
||||
MPI_Comm mpi_comm{team_obj->mpi_comm};
|
||||
|
||||
broadcast_internal<T>(mpi_comm, dest, source, nelems, pe_root);
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
__host__ inline MPI_Op HostInterface::get_mpi_op(ROC_SHMEM_OP Op) {
|
||||
switch (Op) {
|
||||
case ROC_SHMEM_SUM:
|
||||
return MPI_SUM;
|
||||
case ROC_SHMEM_MAX:
|
||||
return MPI_MAX;
|
||||
case ROC_SHMEM_MIN:
|
||||
return MPI_MIN;
|
||||
case ROC_SHMEM_PROD:
|
||||
return MPI_PROD;
|
||||
case ROC_SHMEM_AND:
|
||||
return MPI_BAND;
|
||||
case ROC_SHMEM_OR:
|
||||
return MPI_BOR;
|
||||
case ROC_SHMEM_XOR:
|
||||
return MPI_BXOR;
|
||||
default:
|
||||
fprintf(stderr, "Unknown ROC_SHMEM op MPI conversion %d\n", Op);
|
||||
abort();
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
__host__ inline MPI_Datatype HostInterface::get_mpi_type() {
|
||||
fprintf(stderr, "Unknown or unimplemented datatype \n");
|
||||
}
|
||||
|
||||
#define GET_MPI_TYPE(T, MPI_T) \
|
||||
template <> \
|
||||
__host__ inline MPI_Datatype HostInterface::get_mpi_type<T>() { \
|
||||
return MPI_T; \
|
||||
}
|
||||
|
||||
GET_MPI_TYPE(int, MPI_INT)
|
||||
GET_MPI_TYPE(unsigned int, MPI_UNSIGNED)
|
||||
GET_MPI_TYPE(short, MPI_SHORT)
|
||||
GET_MPI_TYPE(unsigned short, MPI_UNSIGNED_SHORT)
|
||||
GET_MPI_TYPE(long, MPI_LONG)
|
||||
GET_MPI_TYPE(unsigned long, MPI_UNSIGNED_LONG)
|
||||
GET_MPI_TYPE(long long, MPI_LONG_LONG)
|
||||
GET_MPI_TYPE(unsigned long long, MPI_UNSIGNED_LONG_LONG)
|
||||
GET_MPI_TYPE(float, MPI_FLOAT)
|
||||
GET_MPI_TYPE(double, MPI_DOUBLE)
|
||||
GET_MPI_TYPE(char, MPI_CHAR)
|
||||
GET_MPI_TYPE(signed char, MPI_SIGNED_CHAR)
|
||||
GET_MPI_TYPE(unsigned char, MPI_UNSIGNED_CHAR)
|
||||
|
||||
template <typename T>
|
||||
__host__ void HostInterface::amo_add(void* dst, T value, int pe,
|
||||
WindowInfo* window_info) {
|
||||
/*
|
||||
* Most MPI implementations tend to use active messages to implement
|
||||
* MPI_Accumulate. So, to eliminate the potential involvement of the
|
||||
* target PE, we instead use fetch_add and disregard the return value.
|
||||
*/
|
||||
[[maybe_unused]] T ret{amo_fetch_add(dst, value, pe, window_info)};
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
__host__ void HostInterface::amo_cas(void* dst, T value, T cond, int pe,
|
||||
WindowInfo* window_info) {
|
||||
/* Perform the compare and swap and disregard the return value */
|
||||
[[maybe_unused]] T ret{amo_fetch_cas(dst, value, cond, pe, window_info)};
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
__host__ T HostInterface::amo_fetch_add(void* dst, T value, int pe,
|
||||
WindowInfo* window_info) {
|
||||
/* Calculate offset of remote dest from base address of window */
|
||||
MPI_Aint offset{
|
||||
compute_offset(dst, window_info->get_start(), window_info->get_end())};
|
||||
|
||||
/*
|
||||
* Flush the HDP of the remote PE so that the NIC does not
|
||||
* read stale values
|
||||
*/
|
||||
flush_remote_hdp(pe);
|
||||
|
||||
/* Offload remote fetch and op operation to MPI */
|
||||
T ret{};
|
||||
MPI_Win win{window_info->get_win()};
|
||||
MPI_Datatype mpi_type{get_mpi_type<T>()};
|
||||
MPI_Fetch_and_op(&value, &ret, mpi_type, pe, offset, MPI_SUM, win);
|
||||
|
||||
MPI_Win_flush_local(pe, win);
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
__host__ T HostInterface::amo_fetch_cas(void* dst, T value, T cond, int pe,
|
||||
WindowInfo* window_info) {
|
||||
/* Calculate offset of remote dest from base address of window */
|
||||
MPI_Aint offset{
|
||||
compute_offset(dst, window_info->get_start(), window_info->get_end())};
|
||||
|
||||
/*
|
||||
* Flush the HDP of the remote PE so that the NIC does not
|
||||
* read stale values
|
||||
*/
|
||||
flush_remote_hdp(pe);
|
||||
|
||||
/* Offload remote compare and swap operation to MPI */
|
||||
T ret{};
|
||||
MPI_Win win{window_info->get_win()};
|
||||
MPI_Datatype mpi_type{get_mpi_type<T>()};
|
||||
MPI_Compare_and_swap(&value, &cond, &ret, mpi_type, pe, offset, win);
|
||||
|
||||
MPI_Win_flush_local(pe, win);
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
template <typename T, ROC_SHMEM_OP Op>
|
||||
__host__ void HostInterface::to_all_internal(MPI_Comm mpi_comm, T* dest,
|
||||
const T* source, int nreduce) {
|
||||
DPRINTF("Function: host_to_all_internal\n");
|
||||
|
||||
MPI_Op mpi_op{get_mpi_op(Op)};
|
||||
|
||||
MPI_Datatype mpi_type{get_mpi_type<T>()};
|
||||
|
||||
void* send_buf{const_cast<T*>(source)};
|
||||
void* recv_buf{const_cast<T*>(dest)};
|
||||
|
||||
/*
|
||||
* Flush my HDP so that the NIC does not read stale values
|
||||
*/
|
||||
hdp_policy_->hdp_flush();
|
||||
|
||||
/*
|
||||
* Offload the allreduce to MPI
|
||||
*/
|
||||
MPI_Allreduce((dest == source) ? MPI_IN_PLACE : send_buf, recv_buf, nreduce,
|
||||
mpi_type, mpi_op, mpi_comm);
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
template <typename T, ROC_SHMEM_OP Op>
|
||||
__host__ void HostInterface::to_all(T* dest, const T* source, int nreduce,
|
||||
int pe_start, int log_pe_stride,
|
||||
int pe_size, [[maybe_unused]] T* p_wrk,
|
||||
[[maybe_unused]] long* p_sync) {
|
||||
DPRINTF("Function: host_to_all\n");
|
||||
|
||||
/*
|
||||
* Get an MPI communicator for active set of PEs
|
||||
* Note: pe_root is w.r.t. the active set, hence
|
||||
* the MPI communicator contains the root as well.
|
||||
*/
|
||||
MPI_Comm mpi_comm{get_mpi_comm(pe_start, log_pe_stride, pe_size)};
|
||||
|
||||
to_all_internal<T, Op>(mpi_comm, dest, source, nreduce);
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
template <typename T, ROC_SHMEM_OP Op>
|
||||
__host__ void HostInterface::to_all(roc_shmem_team_t team, T* dest,
|
||||
const T* source, int nreduce) {
|
||||
DPRINTF("Function: Team-based host_to_all\n");
|
||||
|
||||
/*
|
||||
* Get the MPI communicator of this team
|
||||
*/
|
||||
Team* team_obj{get_internal_team(team)};
|
||||
MPI_Comm mpi_comm{team_obj->mpi_comm};
|
||||
|
||||
to_all_internal<T, Op>(mpi_comm, dest, source, nreduce);
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
__host__ inline int HostInterface::compare(roc_shmem_cmps cmp, T input_val,
|
||||
T target_val) {
|
||||
int cond_satisfied{0};
|
||||
|
||||
switch (cmp) {
|
||||
case ROC_SHMEM_CMP_EQ:
|
||||
cond_satisfied = (input_val == target_val) ? 1 : 0;
|
||||
break;
|
||||
case ROC_SHMEM_CMP_NE:
|
||||
cond_satisfied = (input_val != target_val) ? 1 : 0;
|
||||
break;
|
||||
case ROC_SHMEM_CMP_GT:
|
||||
cond_satisfied = (input_val > target_val) ? 1 : 0;
|
||||
break;
|
||||
case ROC_SHMEM_CMP_GE:
|
||||
cond_satisfied = (input_val >= target_val) ? 1 : 0;
|
||||
break;
|
||||
case ROC_SHMEM_CMP_LT:
|
||||
cond_satisfied = (input_val < target_val) ? 1 : 0;
|
||||
break;
|
||||
case ROC_SHMEM_CMP_LE:
|
||||
cond_satisfied = (input_val <= target_val) ? 1 : 0;
|
||||
break;
|
||||
default:
|
||||
assert(cmp >= ROC_SHMEM_CMP_EQ && cmp <= ROC_SHMEM_CMP_LE);
|
||||
break;
|
||||
}
|
||||
|
||||
return cond_satisfied;
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
__host__ inline int HostInterface::test_and_compare(MPI_Aint offset,
|
||||
MPI_Datatype mpi_type,
|
||||
roc_shmem_cmps cmp, T val,
|
||||
MPI_Win win) {
|
||||
T fetched_val{};
|
||||
|
||||
/*
|
||||
* Flush the HDP so that the CPU doesn't read stale values
|
||||
*/
|
||||
hdp_policy_->hdp_flush();
|
||||
|
||||
MPI_Fetch_and_op(nullptr, // because no operation happening here
|
||||
&fetched_val, mpi_type, my_pe_, offset, MPI_NO_OP, win);
|
||||
MPI_Win_flush_local(my_pe_, win);
|
||||
|
||||
/*
|
||||
* Compare based on the operation
|
||||
*/
|
||||
return compare(cmp, fetched_val, val);
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
__host__ void HostInterface::wait_until(T* ptr, roc_shmem_cmps cmp, T val,
|
||||
WindowInfo* window_info) {
|
||||
DPRINTF("Function: host_wait_until\n");
|
||||
|
||||
/*
|
||||
* Find the offset of this memory in the window
|
||||
*/
|
||||
MPI_Aint offset{
|
||||
compute_offset(ptr, window_info->get_start(), window_info->get_end())};
|
||||
|
||||
MPI_Datatype mpi_type{get_mpi_type<T>()};
|
||||
MPI_Win win{window_info->get_win()};
|
||||
|
||||
/*
|
||||
* Continuously read the ptr atomically until it satisfies the condition
|
||||
*/
|
||||
while (1) {
|
||||
int cond_satisfied{test_and_compare(offset, mpi_type, cmp, val, win)};
|
||||
|
||||
if (cond_satisfied) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
__host__ size_t status_entry(size_t nelems,
|
||||
const int *status,
|
||||
bool* done_flags) {
|
||||
size_t i{0};
|
||||
size_t pos{SIZE_MAX};
|
||||
while (i < nelems) {
|
||||
if (status[i]) {
|
||||
done_flags[i] = 1;
|
||||
} else {
|
||||
pos = min(i, pos);
|
||||
}
|
||||
i++;
|
||||
}
|
||||
return pos;
|
||||
}
|
||||
|
||||
__host__ size_t status_entry(size_t nelems,
|
||||
const int *status) {
|
||||
size_t i{0};
|
||||
while (i < nelems) {
|
||||
if (status[i] == 0) {
|
||||
return i;
|
||||
}
|
||||
i++;
|
||||
}
|
||||
return i;
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
__host__ size_t HostInterface::wait_until_any(T* ptr, size_t nelems,
|
||||
const int *status,
|
||||
roc_shmem_cmps cmp, T val,
|
||||
WindowInfo* window_info) {
|
||||
DPRINTF("Function: host_wait_until_any\n");
|
||||
|
||||
// zero nelems error condition
|
||||
if (!nelems) {
|
||||
return SIZE_MAX;
|
||||
}
|
||||
|
||||
size_t pos{status_entry(nelems, status)};
|
||||
|
||||
// invalid (empty) status array error condition
|
||||
if (pos == nelems) {
|
||||
return SIZE_MAX;
|
||||
}
|
||||
|
||||
while (true) {
|
||||
for (size_t i{pos}; i < nelems; i++) {
|
||||
// skip entries marked with non-zero status
|
||||
if (status[i]) {
|
||||
continue;
|
||||
}
|
||||
if (test(ptr + i, cmp, val, window_info)) {
|
||||
return i;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
__host__ void HostInterface::wait_until_all(T* ptr, size_t nelems,
|
||||
const int *status,
|
||||
roc_shmem_cmps cmp, T val,
|
||||
WindowInfo* window_info) {
|
||||
DPRINTF("Function: host_wait_until_all\n");
|
||||
|
||||
// zero nelems error condition
|
||||
if (!nelems) {
|
||||
return;
|
||||
}
|
||||
|
||||
size_t pos{status_entry(nelems, status)};
|
||||
|
||||
// invalid (empty) status array error condition
|
||||
if (pos == nelems) {
|
||||
return;
|
||||
}
|
||||
|
||||
for (size_t i{pos}; i < nelems; i++) {
|
||||
if (status[i]) {
|
||||
continue;
|
||||
}
|
||||
while (!test(ptr + i, cmp, val, window_info)) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
__host__ size_t HostInterface::wait_until_some(T* ptr, size_t nelems,
|
||||
size_t* indices,
|
||||
const int *status,
|
||||
roc_shmem_cmps cmp, T val,
|
||||
WindowInfo* window_info) {
|
||||
DPRINTF("Function: host_wait_until_some\n");
|
||||
|
||||
// zero nelems error condition
|
||||
if (!nelems) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
size_t pos{status_entry(nelems, status)};
|
||||
|
||||
// invalid (empty) status array error condition
|
||||
if (pos == nelems) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
bool done {false};
|
||||
size_t ncompleted {0};
|
||||
while (!done) {
|
||||
for (size_t i{pos}; i < nelems; i++) {
|
||||
// skip entries marked with non-zero status
|
||||
if (status[i]) {
|
||||
continue;
|
||||
}
|
||||
if (test(ptr + i, cmp, val, window_info)) {
|
||||
done = true;
|
||||
indices[ncompleted] = i;
|
||||
ncompleted++;
|
||||
}
|
||||
}
|
||||
}
|
||||
return ncompleted;
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
__host__ void HostInterface::wait_until_all_vector(T* ptr, size_t nelems,
|
||||
const int *status,
|
||||
roc_shmem_cmps cmp, T* vals,
|
||||
WindowInfo* window_info) {
|
||||
DPRINTF("Function: host_wait_until_all_vector\n");
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
__host__ size_t HostInterface::wait_until_any_vector(T* ptr, size_t nelems,
|
||||
const int *status,
|
||||
roc_shmem_cmps cmp, T* vals,
|
||||
WindowInfo* window_info) {
|
||||
DPRINTF("Function: host_wait_until_any_vector\n");
|
||||
return 0;
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
__host__ size_t HostInterface::wait_until_some_vector(T* ptr, size_t nelems,
|
||||
size_t* indices,
|
||||
const int *status,
|
||||
roc_shmem_cmps cmp, T* vals,
|
||||
WindowInfo* window_info) {
|
||||
DPRINTF("Function: host_wait_until_some_vector\n");
|
||||
return 0;
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
__host__ int HostInterface::test(T* ptr, roc_shmem_cmps cmp, T val,
|
||||
WindowInfo* window_info) {
|
||||
DPRINTF("Function: host_test\n");
|
||||
|
||||
/*
|
||||
* Find the offset of this memory in the window
|
||||
*/
|
||||
MPI_Aint offset{
|
||||
compute_offset(ptr, window_info->get_start(), window_info->get_end())};
|
||||
|
||||
MPI_Datatype mpi_type{get_mpi_type<T>()};
|
||||
|
||||
return test_and_compare(offset, mpi_type, cmp, val, window_info->get_win());
|
||||
}
|
||||
|
||||
} // namespace rocshmem
|
||||
|
||||
#endif // LIBRARY_SRC_HOST_HOST_TEMPLATES_HPP_
|
||||
Reference in New Issue
Block a user