AI/rocm-systems
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

unify env variables and use DPRINTF (#89)

Browse Source 
* unify handling of env variables

create a class containing all (most?) environment variables used by rocshmem and an object that is instatiated
before library_init, since some of the environment variables need to be
set before we start the bootstraping process.

This allows us to remove two files from the bootstrap directory.

* replace INFO and TRACE macros with DPRINTF

to be more consistent with the rest of the rocSHMEM code

[ROCm/rocshmem commit: db74307195]
This commit is contained in:
Edgar Gabriel
2025-04-29 06:05:25 -05:00
committed by GitHub
parent 8f135af156
commit 2e6fed8e79
12 changed files with 113 additions and 227 deletions
Download Patch File Download Diff File Expand all files Collapse all files
projects/rocshmem/src/bootstrap/CMakeLists.txt
-1
View File
@@ -30,6 +30,5 @@ target_sources(
PRIVATE
socket.cpp
bootstrap.cpp
env.cpp
utils.cpp
)
projects/rocshmem/src/bootstrap/bootstrap.cpp
+15 -14
View File
@@ -32,6 +32,7 @@
#include "bootstrap.hpp"
#include "utils.hpp"
#include "../util.hpp"
#include "socket.hpp"
namespace rocshmem {
@@ -39,12 +40,12 @@ namespace rocshmem {
static void setFilesLimit() {
rlimit filesLimit;
if (getrlimit(RLIMIT_NOFILE, &filesLimit) != 0) {
INFO("getrlimit failed\n");
DPRINTF("getrlimit failed\n");
return;
}
filesLimit.rlim_cur = filesLimit.rlim_max;
if (setrlimit(RLIMIT_NOFILE, &filesLimit) != 0) {
INFO("setrlimit failed\n");
DPRINTF("setrlimit failed\n");
return;
}
}
@@ -193,7 +194,7 @@ void TcpBootstrap::Impl::initialize(const rocshmem_uniqueid_t& uniqueId, int64_t
char line[MAX_IF_NAME_SIZE + 1];
SocketToString(&uniqueId_.addr, line);
TRACE("rank %d nranks %d - connecting to %s\n", rank_, nRanks_, line);
DPRINTF("rank %d nranks %d - connecting to %s\n", rank_, nRanks_, line);
establishConnections(timeoutSec);
}
@@ -305,28 +306,28 @@ void TcpBootstrap::Impl::bootstrapRoot() {
std::memset(rankAddressesRoot.data(), 0, sizeof(SocketAddress) * nRanks_);
setFilesLimit();
TRACE("BEGIN bootstrapRoot\n");
DPRINTF("BEGIN bootstrapRoot\n");
/* Receive addresses from all ranks */
do {
int rank;
getRemoteAddresses(listenSockRoot_.get(), rankAddresses, rankAddressesRoot, rank);
++numCollected;
TRACE("Received connect from rank %d total %d/%d\n", rank, numCollected, nRanks_);
DPRINTF("Received connect from rank %d total %d/%d\n", rank, numCollected, nRanks_);
} while (numCollected < nRanks_ && (!abortFlag_ || *abortFlag_ == 0));
if (abortFlag_ && *abortFlag_) {
TRACE("ABORTED\n");
DPRINTF("ABORTED\n");
return;
}
TRACE("COLLECTED ALL %d HANDLES\n", nRanks_);
DPRINTF("COLLECTED ALL %d HANDLES\n", nRanks_);
// Send the connect handle for the next rank in the AllGather ring
for (int peer = 0; peer < nRanks_; ++peer) {
sendHandleToPeer(peer, rankAddresses, rankAddressesRoot);
}
TRACE("DONE bootstrapRoot\n");
DPRINTF("DONE bootstrapRoot\n");
}
void TcpBootstrap::Impl::netInit(std::string ipPortPair, std::string interface,
@@ -361,7 +362,7 @@ void TcpBootstrap::Impl::netInit(std::string ipPortPair, std::string interface,
char line[SOCKET_NAME_MAXLEN + MAX_IF_NAME_SIZE + 2];
std::sprintf(line, " %s:", netIfName);
SocketToString(&netIfAddr, line + strlen(line));
TRACE("TcpBootstrap : Using%s", line);
DPRINTF("TcpBootstrap : Using%s", line);
}
#define TIMEOUT(__exp) \
@@ -382,7 +383,7 @@ void TcpBootstrap::Impl::establishConnections(int64_t timeoutSec) {
SocketAddress nextAddr;
ExtInfo info;
TRACE("establishConnections: rank %d nranks %d\n", rank_, nRanks_);
DPRINTF("establishConnections: rank %d nranks %d\n", rank_, nRanks_);
auto getLeftTime = [&]() {
if (connectionTimeoutUs < 0) {
@@ -417,7 +418,7 @@ void TcpBootstrap::Impl::establishConnections(int64_t timeoutSec) {
timespec tv;
tv.tv_sec = rank / 1000;
tv.tv_nsec = 1000000 * (rank % 1000);
TRACE("rank %d delaying connection to root by %ld sec %ld nsec\n", rank,
DPRINTF("rank %d delaying connection to root by %ld sec %ld nsec\n", rank,
tv.tv_sec, tv.tv_nsec);
(void)nanosleep(&tv, NULL);
};
@@ -455,7 +456,7 @@ void TcpBootstrap::Impl::establishConnections(int64_t timeoutSec) {
peerCommAddresses_[rank_] = listenSock_->getAddr();
allGather(peerCommAddresses_.data(), sizeof(SocketAddress));
TRACE("rank %d nranks %d - DONE\n", rank_, nRanks_);
DPRINTF("rank %d nranks %d - DONE\n", rank_, nRanks_);
}
int TcpBootstrap::Impl::getNranksPerNode() {
@@ -485,7 +486,7 @@ void TcpBootstrap::Impl::allGather(void* allData, int size) {
int rank = rank_;
int nRanks = nRanks_;
TRACE("allGather: rank %d nranks %d size %d\n", rank, nRanks, size);
DPRINTF("allGather: rank %d nranks %d size %d\n", rank, nRanks, size);
/* Simple ring based AllGather
* At each step i receive data from (rank-i-1) from left
@@ -501,7 +502,7 @@ void TcpBootstrap::Impl::allGather(void* allData, int size) {
netRecv(ringRecvSocket_.get(), data + rSlice * size, size);
}
TRACE("allGather: rank %d nranks %d size %d - DONE\n", rank, nRanks, size);
DPRINTF("allGather: rank %d nranks %d size %d - DONE\n", rank, nRanks, size);
}
std::shared_ptr<Socket> TcpBootstrap::Impl::getPeerSendSocket(int peer, int tag) {
projects/rocshmem/src/bootstrap/env.cpp
-95
View File
@@ -1,95 +0,0 @@
/******************************************************************************
* Copyright (c) Microsoft Corporation.
* Modifications Copyright (c) Advanced Micro Devices, Inc. All rights reserved.
*
* SPDX-License-Identifier: MIT
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to
* deal in the Software without restriction, including without limitation the
* rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
* sell copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in
* all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
* IN THE SOFTWARE.
*****************************************************************************/
#include <cstdlib>
#include <type_traits>
#include "env.hpp"
#include "utils.hpp"
template <typename T>
T readEnv(const std::string &envName, const T &defaultValue) {
const char *envCstr = getenv(envName.c_str());
if (envCstr == nullptr) return defaultValue;
if constexpr (std::is_same_v<T, int>) {
return atoi(envCstr);
} else if constexpr (std::is_same_v<T, bool>) {
return (std::string(envCstr) != "0");
}
return T(envCstr);
}
template <typename T>
void readAndSetEnv(const std::string &envName, T &env) {
const char *envCstr = getenv(envName.c_str());
if (envCstr == nullptr) return;
if constexpr (std::is_same_v<T, int>) {
env = atoi(envCstr);
} else if constexpr (std::is_same_v<T, bool>) {
env = (std::string(envCstr) != "0");
} else {
env = std::string(envCstr);
}
}
template <typename T>
void logEnv(const std::string &envName, const T &env) {
if (!getenv(envName.c_str())) return;
INFO("%s=%d", envName.c_str(), env);
}
template <>
void logEnv(const std::string &envName, const std::string &env) {
if (!getenv(envName.c_str())) return;
INFO("%s=%s", envName.c_str(), env.c_str());
}
namespace rocshmem {
Env::Env()
: debug(readEnv<std::string>("ROCSHMEM_DEBUG", "")),
debugSubsys(readEnv<std::string>("ROCSHMEM_DEBUG_SUBSYS", "")),
debugFile(readEnv<std::string>("ROCSHMEM_DEBUG_FILE", "")),
hostid(readEnv<std::string>("ROCSHMEM_HOSTID", "")),
socketFamily(readEnv<std::string>("ROCSHMEM_SOCKET_FAMILY", "")),
socketIfname(readEnv<std::string>("ROCSHMEM_SOCKET_IFNAME", "")) {}
std::shared_ptr<Env> env() {
static std::shared_ptr<Env> globalEnv = std::shared_ptr<Env>(new Env());
static bool logged = false;
if (!logged) {
logged = true;
// cannot log inside the constructor because of circular dependency
logEnv("ROCSHMEM_DEBUG", globalEnv->debug);
logEnv("ROCSHMEM_DEBUG_SUBSYS", globalEnv->debugSubsys);
logEnv("ROCSHMEM_DEBUG_FILE", globalEnv->debugFile);
logEnv("ROCSHMEM_HOSTID", globalEnv->hostid);
logEnv("ROCSHMEM_SOCKET_FAMILY", globalEnv->socketFamily);
logEnv("ROCSHMEM_SOCKET_IFNAME", globalEnv->socketIfname);
}
return globalEnv;
}
} // namespace rocshmem
projects/rocshmem/src/bootstrap/env.hpp
-59
View File
@@ -1,59 +0,0 @@
/******************************************************************************
* Copyright (c) Microsoft Corporation.
* Modifications Copyright (c) Advanced Micro Devices, Inc. All rights reserved.
*
* SPDX-License-Identifier: MIT
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to
* deal in the Software without restriction, including without limitation the
* rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
* sell copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in
* all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
* IN THE SOFTWARE.
*****************************************************************************/
#ifndef ROCSHMEM_ENV_HPP_
#define ROCSHMEM_ENV_HPP_
#include <memory>
#include <string>
namespace rocshmem {
class Env;
/// Get the environment.
/// @return A reference to the global environment object.
std::shared_ptr<Env> env();
/// The constructor reads environment variables and sets the corresponding fields.
/// Use the @ref env() function to get the environment object.
class Env {
public:
const std::string debug;
const std::string debugSubsys;
const std::string debugFile;
const std::string hostid;
const std::string socketFamily;
const std::string socketIfname;
private:
Env();
friend std::shared_ptr<Env> env();
};
} // namespace rocshmem
#endif // ROCSHMEM_ENV_HPP_
projects/rocshmem/src/bootstrap/socket.cpp
+24 -23
View File
@@ -32,10 +32,11 @@
#include <unistd.h>
#include <fstream>
#include <cstring>
#include "socket.hpp"
#include "utils.hpp"
#include "env.hpp"
#include "../util.hpp"
namespace rocshmem {
@@ -85,7 +86,7 @@ static uint16_t socketToPort(union SocketAddress* addr) {
/* Allow the user to force the IPv4/IPv6 interface selection */
static int envSocketFamily(void) {
int family = -1; // Family selection is not forced, will use first one found
const std::string& socketFamily = env()->socketFamily;
const std::string& socketFamily = rocshmem_env_.get_bootstrap_socket_family();
if (socketFamily == "") return family;
if (socketFamily == "AF_INET")
@@ -97,7 +98,7 @@ static int envSocketFamily(void) {
static int findInterfaces(const char* prefixList, char* names, union SocketAddress* addrs,
int sock_family, int maxIfNameSize, int maxIfs) {
#ifdef ROCSHMEM_ENABLE_TRACE
#ifdef DEBUG
char line[SOCKET_NAME_MAXLEN + 1];
#endif
struct netIf userIfs[MAX_IFS];
@@ -117,7 +118,7 @@ static int findInterfaces(const char* prefixList, char* names, union SocketAddre
int family = interface->ifa_addr->sa_family;
if (family != AF_INET && family != AF_INET6) continue;
TRACE("Found interface %s:%s\n", interface->ifa_name,
DPRINTF("Found interface %s:%s\n", interface->ifa_name,
SocketToString((union SocketAddress*)interface->ifa_addr, line));
/* Allow the caller to force the socket family type */
@@ -149,7 +150,7 @@ static int findInterfaces(const char* prefixList, char* names, union SocketAddre
strncpy(names + found * maxIfNameSize, interface->ifa_name, maxIfNameSize);
// Store the IP address
int salen = (family == AF_INET) ? sizeof(struct sockaddr_in) : sizeof(struct sockaddr_in6);
memcpy(addrs + found, interface->ifa_addr, salen);
std::memcpy(addrs + found, interface->ifa_addr, salen);
found++;
}
}
@@ -203,7 +204,7 @@ static bool matchSubnet(struct ifaddrs local_if, union SocketAddress* remote) {
int FindInterfaceMatchSubnet(char* ifNames, union SocketAddress* localAddrs, union SocketAddress* remoteAddr,
int ifNameMaxSize, int maxIfs) {
#ifdef ROCSHMEM_ENABLE_TRACE
#ifdef DEBUG
char line[SOCKET_NAME_MAXLEN + 1];
#endif
char line_a[SOCKET_NAME_MAXLEN + 1];
@@ -224,12 +225,12 @@ int FindInterfaceMatchSubnet(char* ifNames, union SocketAddress* localAddrs, uni
// Store the local IP address
int salen = (family == AF_INET) ? sizeof(struct sockaddr_in) : sizeof(struct sockaddr_in6);
memcpy(localAddrs + found, interface->ifa_addr, salen);
std::memcpy(localAddrs + found, interface->ifa_addr, salen);
// Store the interface name
strncpy(ifNames + found * ifNameMaxSize, interface->ifa_name, ifNameMaxSize);
TRACE("NET : Found interface %s:%s in the same subnet as remote address %s\n",
DPRINTF("NET : Found interface %s:%s in the same subnet as remote address %s\n",
interface->ifa_name, SocketToString(localAddrs + found, line), SocketToString(remoteAddr, line_a));
found++;
if (found == maxIfs) break;
@@ -273,13 +274,13 @@ void SocketGetAddrFromString(union SocketAddress* ua, const char* ip_port_pair)
// use the first
if (p->ai_family == AF_INET) {
struct sockaddr_in& sin = ua->sin;
memcpy(&sin, p->ai_addr, sizeof(struct sockaddr_in));
std::memcpy(&sin, p->ai_addr, sizeof(struct sockaddr_in));
sin.sin_family = AF_INET; // IPv4
// inet_pton(AF_INET, ni.prefix, &(sin.sin_addr)); // IP address
sin.sin_port = htons(ni.port); // port
} else if (p->ai_family == AF_INET6) {
struct sockaddr_in6& sin6 = ua->sin6;
memcpy(&sin6, p->ai_addr, sizeof(struct sockaddr_in6));
std::memcpy(&sin6, p->ai_addr, sizeof(struct sockaddr_in6));
sin6.sin6_family = AF_INET6; // IPv6
sin6.sin6_port = htons(ni.port); // port
sin6.sin6_flowinfo = 0; // needed by IPv6, but possibly obsolete
@@ -333,13 +334,13 @@ int FindInterfaces(char* ifNames, union SocketAddress* ifAddrs, int ifNameMaxSiz
int sock_family = envSocketFamily();
// User specified interface
const std::string& socketIfname = env()->socketIfname;
const std::string& socketIfname = rocshmem_env_.get_bootstrap_socket_ifname();
if (inputIfName) {
TRACE("using iterface %s", inputIfName);
DPRINTF("using iterface %s", inputIfName);
nIfs = findInterfaces(inputIfName, ifNames, ifAddrs, sock_family, ifNameMaxSize, maxIfs);
} else if (socketIfname != "") {
// Specified by user : find or fail
if (shownIfName++ == 0) TRACE ("ROCSHMEM_SOCKET_IFNAME set to %s", socketIfname.c_str());
if (shownIfName++ == 0) DPRINTF ("ROCSHMEM_SOCKET_IFNAME set to %s", socketIfname.c_str());
nIfs = findInterfaces(socketIfname.c_str(), ifNames, ifAddrs, sock_family,
ifNameMaxSize, maxIfs);
} else {
@@ -371,7 +372,7 @@ Socket::Socket(const SocketAddress* addr, uint64_t magic, enum SocketType type,
if (addr) {
/* IPv4/IPv6 support */
int family;
memcpy(&addr_, addr, sizeof(union SocketAddress));
std::memcpy(&addr_, addr, sizeof(union SocketAddress));
family = addr_.sa.sa_family;
if (family != AF_INET && family != AF_INET6) {
char line[SOCKET_NAME_MAXLEN + 1];
@@ -441,7 +442,7 @@ void Socket::bind() {
return;
}
if (remainSecs > 0) {
TRACE("No available ephemeral ports found, will retry after 1 second");
DPRINTF("No available ephemeral ports found, will retry after 1 second");
sleep(1);
remainSecs--;
} else {
@@ -460,11 +461,11 @@ void Socket::bind() {
}
void Socket::bindAndListen() {
#ifdef ROCSHMEM_ENABLE_TRACE
#ifdef DEBUG
char line[SOCKET_NAME_MAXLEN + 1];
#endif
bind();
TRACE("Listening on socket %s\n", SocketToString(&addr_, line));
DPRINTF("Listening on socket %s\n", SocketToString(&addr_, line));
/* Put the socket in listen mode
* NB: The backlog will be silently truncated to the value in /proc/sys/net/core/somaxconn
@@ -477,7 +478,7 @@ void Socket::bindAndListen() {
}
void Socket::connect(int64_t timeout) {
#ifdef ROCSHMEM_ENABLE_TRACE
#ifdef DEBUG
char line[SOCKET_NAME_MAXLEN + 1];
#endif
Timer timer;
@@ -492,10 +493,10 @@ void Socket::connect(int64_t timeout) {
ERROR("wrong socket state %d\n", state_);
return;
}
TRACE("Connecting to socket %s \n", SocketToString(&addr_, line));
DPRINTF("Connecting to socket %s \n", SocketToString(&addr_, line));
if (setsockopt(fd_, IPPROTO_TCP, TCP_NODELAY, (char*)&one, sizeof(int)) != 0) {
INFO("setsockopt(TCP_NODELAY) failed, errno %d\n", errno);
DPRINTF("setsockopt(TCP_NODELAY) failed, errno %d\n", errno);
return;
}
@@ -642,7 +643,7 @@ void Socket::tryAccept() {
} else {
usleep(SLEEP_INT);
if (++acceptRetries_ % 1000 == 0)
INFO("tryAccept: Call to try accept returned %s, retrying", strerror(errno));
DPRINTF("tryAccept: Call to try accept returned %s, retrying", strerror(errno));
}
}
@@ -686,7 +687,7 @@ void Socket::startConnect() {
return;
} else if (errno == ECONNREFUSED || errno == ETIMEDOUT) {
usleep(SLEEP_INT);
if (++connectRetries_ % 1000 == 0) INFO("Call to connect returned %s, retrying", strerror(errno));
if (++connectRetries_ % 1000 == 0) DPRINTF("Call to connect returned %s, retrying", strerror(errno));
return;
} else {
char line[SOCKET_NAME_MAXLEN + 1];
@@ -725,7 +726,7 @@ void Socket::pollConnect() {
state_ = SocketStateConnected;
} else if (ret == ECONNREFUSED || ret == ETIMEDOUT) {
if (++connectRetries_ % 1000 == 0) {
INFO("Call to connect returned %s, retrying", strerror(errno));
DPRINTF("Call to connect returned %s, retrying", strerror(errno));
}
usleep(SLEEP_INT);
projects/rocshmem/src/bootstrap/utils.cpp
+4 -4
View File
@@ -34,7 +34,7 @@
#include <iostream>
#include "utils.hpp"
#include "env.hpp"
#include "../util.hpp"
constexpr char HOSTID_FILE[32] = "/proc/sys/kernel/random/boot_id";
@@ -101,7 +101,7 @@ uint64_t computeHostHash(void) {
std::string hostName = getHostName(hashLen, '\0');
strncpy(hostHash, hostName.c_str(), hostName.size());
std::string hostid = env()->hostid;
std::string hostid = rocshmem_env_.get_bootstrap_hostid();
if (hostid != "") {
strncpy(hostHash, hostid.c_str(), hashLen);
} else if (hostName.size() < hashLen) {
@@ -113,7 +113,7 @@ uint64_t computeHostHash(void) {
// Make sure the string is terminated
hostHash[sizeof(hostHash) - 1] = '\0';
TRACE("unique hostname '%s'", hostHash);
DPRINTF("unique hostname '%s'", hostHash);
return getHash(hostHash, strlen(hostHash));
}
@@ -141,7 +141,7 @@ uint64_t computePidHash(void) {
if (len < 0) len = 0;
pname[plen + len] = '\0';
TRACE("unique PID '%s'", pname);
DPRINTF("unique PID '%s'", pname);
return getHash(pname, strlen(pname));
}
projects/rocshmem/src/bootstrap/utils.hpp
-12
View File
@@ -33,18 +33,6 @@
#define ERROR(...) { fprintf(stderr, __VA_ARGS__); abort(); }
#ifdef ROCSHMEM_ENABLE_TRACE
#define TRACE(...) printf(__VA_ARGS__)
#else
#define TRACE(...)
#endif
#if defined ROCSHMEM_ENABLE_INFO
#define INFO(FLAGS, ...)printf(__VA_ARGS__)
#else
#define INFO(...)
#endif
namespace rocshmem {
struct Timer {
projects/rocshmem/src/ipc_policy.cpp
+1 -1
View File
@@ -109,7 +109,7 @@ __host__ void IpcOnImpl::ipcHostInit(int my_pe, const HEAP_BASES_T &heap_bases,
*/
free(vec_ipc_handle);
if (0 == rocshmem_env_config.ro_disable_ipc) {
if (0 == rocshmem_env_.get_ro_disable_ipc()) {
int thread_comm_rank;
CHECK_HIP(hipMalloc(reinterpret_cast<void**>(&pes_with_ipc_avail), shm_size * sizeof(int)));
projects/rocshmem/src/reverse_offload/mpi_transport.cpp
+3 -1
View File
@@ -590,6 +590,8 @@ std::unique_ptr<MPI_Request[]> MPITransport::raw_requests() {
}
void MPITransport::progress() {
static int progress_delay = rocshmem_env_.get_ro_progress_delay();
if (requests.size() == 0) {
const int tag{1000};
int flag{0};
@@ -597,7 +599,7 @@ void MPITransport::progress() {
// Slowing the progress engine down a bit avoid hammering the memory subsystem.
// This leads to significant performance benefits
usleep (rocshmem_env_config.ro_progress_delay);
usleep (progress_delay);
NET_CHECK(MPI_Iprobe(MPI_ANY_SOURCE, tag, ro_net_comm_world, &flag, &status));
} else {
DPRINTF("Testing all outstanding requests (%zu)\n", requests.size());
projects/rocshmem/src/rocshmem.cpp
+1 -10
View File
@@ -86,8 +86,6 @@ rocshmem_ctx_t ROCSHMEM_HOST_CTX_DEFAULT;
rocm_init();
rocshmem_env_config_init();
#ifdef USE_RO
CHECK_HIP(hipHostMalloc(&backend, sizeof(ROBackend)));
backend = new (backend) ROBackend(comm);
@@ -155,14 +153,7 @@ rocshmem_ctx_t ROCSHMEM_HOST_CTX_DEFAULT;
TcpBootstrap bootstr(attr->rank, attr->nranks);
int timeout = 5;
char *value;
value = getenv("ROCSHMEM_BOOTSTRAP_TIMEOUT");
if (value != nullptr) {
timeout = atoi(value);
}
bootstr.initialize(attr->uid, timeout);
bootstr.initialize(attr->uid, rocshmem_env_.get_bootstrap_timeout());
int *inc_ranks = new int[attr->nranks];
inc_ranks[attr->rank] = world_rank;
projects/rocshmem/src/util.cpp
+48 -4
View File
@@ -34,6 +34,8 @@ namespace rocshmem {
__constant__ int* print_lock;
rocshmem_env_config rocshmem_env_;
typedef struct device_agent {
hsa_agent_t agent;
hsa_amd_memory_pool_t pool;
@@ -121,21 +123,63 @@ void rocm_memory_lock_to_fine_grain(void* ptr, size_t size, void** gpu_ptr,
}
}
struct rocshmem_env_config_t rocshmem_env_config;
void rocshmem_env_config_init(void) {
rocshmem_env_config::rocshmem_env_config() {
char* env_value = NULL;
env_value = getenv("ROCSHMEM_RO_DISABLE_IPC");
if (NULL != env_value) {
rocshmem_env_config.ro_disable_ipc = atoi(env_value);
ro_disable_ipc = atoi(env_value);
}
env_value = getenv("ROCSHMEM_RO_PROGRESS_DELAY");
if (nullptr != env_value) {
rocshmem_env_config.ro_progress_delay = atoi(env_value);
ro_progress_delay = atoi(env_value);
}
env_value = getenv("ROCSHMEM_BOOTSTRAP_TIMEOUT");
if (nullptr != env_value) {
bootstrap_timeout = atoi(env_value);
}
env_value = getenv("ROCSHMEM_BOOTSTRAP_HOSTID");
if (nullptr != env_value) {
bootstrap_hostid = std::string(env_value);
}
env_value = getenv("ROCSHMEM_BOOTSTRAP_SOCKET_FAMILY");
if (nullptr != env_value) {
bootstrap_socket_family = std::string(env_value);
}
env_value = getenv("ROCSHMEM_BOOTSTRAP_SOCKET_IFNAME");
if (nullptr != env_value) {
bootstrap_socket_ifname = std::string(env_value);
}
}
int rocshmem_env_config::get_ro_disable_ipc() {
return ro_disable_ipc;
}
int rocshmem_env_config::get_ro_progress_delay() {
return ro_progress_delay;
}
int rocshmem_env_config::get_bootstrap_timeout() {
return bootstrap_timeout;
}
std::string rocshmem_env_config::get_bootstrap_hostid() {
return bootstrap_hostid;
}
std::string rocshmem_env_config::get_bootstrap_socket_family() {
return bootstrap_socket_family;
}
std::string rocshmem_env_config::get_bootstrap_socket_ifname() {
return bootstrap_socket_ifname;
}
} // namespace rocshmem
projects/rocshmem/src/util.hpp
+17 -3
View File
@@ -268,13 +268,27 @@ int rocm_init();
void rocm_memory_lock_to_fine_grain(void* ptr, size_t size, void** gpu_ptr,
int gpu_id);
struct rocshmem_env_config_t {
class rocshmem_env_config {
public:
rocshmem_env_config();
int get_ro_disable_ipc();
int get_ro_progress_delay();
int get_bootstrap_timeout();
std::string get_bootstrap_hostid();
std::string get_bootstrap_socket_family();
std::string get_bootstrap_socket_ifname();
private:
int ro_disable_ipc = 0;
int ro_progress_delay = 3;
int bootstrap_timeout = 5;
std::string bootstrap_hostid;
std::string bootstrap_socket_family;
std::string bootstrap_socket_ifname;
};
extern struct rocshmem_env_config_t rocshmem_env_config;
void rocshmem_env_config_init(void);
extern rocshmem_env_config rocshmem_env_;
} // namespace rocshmem