From db74307195d1c37035010ec82818b313a12f7ca8 Mon Sep 17 00:00:00 2001 From: Edgar Gabriel Date: Tue, 29 Apr 2025 06:05:25 -0500 Subject: [PATCH] unify env variables and use DPRINTF (#89) * unify handling of env variables create a class containing all (most?) environment variables used by rocshmem and an object that is instatiated before library_init, since some of the environment variables need to be set before we start the bootstraping process. This allows us to remove two files from the bootstrap directory. * replace INFO and TRACE macros with DPRINTF to be more consistent with the rest of the rocSHMEM code --- src/bootstrap/CMakeLists.txt | 1 - src/bootstrap/bootstrap.cpp | 29 ++++---- src/bootstrap/env.cpp | 95 --------------------------- src/bootstrap/env.hpp | 59 ----------------- src/bootstrap/socket.cpp | 47 ++++++------- src/bootstrap/utils.cpp | 8 +-- src/bootstrap/utils.hpp | 12 ---- src/ipc_policy.cpp | 2 +- src/reverse_offload/mpi_transport.cpp | 4 +- src/rocshmem.cpp | 11 +--- src/util.cpp | 52 +++++++++++++-- src/util.hpp | 20 +++++- 12 files changed, 113 insertions(+), 227 deletions(-) delete mode 100644 src/bootstrap/env.cpp delete mode 100644 src/bootstrap/env.hpp diff --git a/src/bootstrap/CMakeLists.txt b/src/bootstrap/CMakeLists.txt index 9c6cab4c2a..82ecadc3ed 100644 --- a/src/bootstrap/CMakeLists.txt +++ b/src/bootstrap/CMakeLists.txt @@ -30,6 +30,5 @@ target_sources( PRIVATE socket.cpp bootstrap.cpp - env.cpp utils.cpp ) diff --git a/src/bootstrap/bootstrap.cpp b/src/bootstrap/bootstrap.cpp index 58d01e39fb..54311a462a 100644 --- a/src/bootstrap/bootstrap.cpp +++ b/src/bootstrap/bootstrap.cpp @@ -32,6 +32,7 @@ #include "bootstrap.hpp" #include "utils.hpp" +#include "../util.hpp" #include "socket.hpp" namespace rocshmem { @@ -39,12 +40,12 @@ namespace rocshmem { static void setFilesLimit() { rlimit filesLimit; if (getrlimit(RLIMIT_NOFILE, &filesLimit) != 0) { - INFO("getrlimit failed\n"); + DPRINTF("getrlimit failed\n"); return; } filesLimit.rlim_cur = filesLimit.rlim_max; if (setrlimit(RLIMIT_NOFILE, &filesLimit) != 0) { - INFO("setrlimit failed\n"); + DPRINTF("setrlimit failed\n"); return; } } @@ -193,7 +194,7 @@ void TcpBootstrap::Impl::initialize(const rocshmem_uniqueid_t& uniqueId, int64_t char line[MAX_IF_NAME_SIZE + 1]; SocketToString(&uniqueId_.addr, line); - TRACE("rank %d nranks %d - connecting to %s\n", rank_, nRanks_, line); + DPRINTF("rank %d nranks %d - connecting to %s\n", rank_, nRanks_, line); establishConnections(timeoutSec); } @@ -305,28 +306,28 @@ void TcpBootstrap::Impl::bootstrapRoot() { std::memset(rankAddressesRoot.data(), 0, sizeof(SocketAddress) * nRanks_); setFilesLimit(); - TRACE("BEGIN bootstrapRoot\n"); + DPRINTF("BEGIN bootstrapRoot\n"); /* Receive addresses from all ranks */ do { int rank; getRemoteAddresses(listenSockRoot_.get(), rankAddresses, rankAddressesRoot, rank); ++numCollected; - TRACE("Received connect from rank %d total %d/%d\n", rank, numCollected, nRanks_); + DPRINTF("Received connect from rank %d total %d/%d\n", rank, numCollected, nRanks_); } while (numCollected < nRanks_ && (!abortFlag_ || *abortFlag_ == 0)); if (abortFlag_ && *abortFlag_) { - TRACE("ABORTED\n"); + DPRINTF("ABORTED\n"); return; } - TRACE("COLLECTED ALL %d HANDLES\n", nRanks_); + DPRINTF("COLLECTED ALL %d HANDLES\n", nRanks_); // Send the connect handle for the next rank in the AllGather ring for (int peer = 0; peer < nRanks_; ++peer) { sendHandleToPeer(peer, rankAddresses, rankAddressesRoot); } - TRACE("DONE bootstrapRoot\n"); + DPRINTF("DONE bootstrapRoot\n"); } void TcpBootstrap::Impl::netInit(std::string ipPortPair, std::string interface, @@ -361,7 +362,7 @@ void TcpBootstrap::Impl::netInit(std::string ipPortPair, std::string interface, char line[SOCKET_NAME_MAXLEN + MAX_IF_NAME_SIZE + 2]; std::sprintf(line, " %s:", netIfName); SocketToString(&netIfAddr, line + strlen(line)); - TRACE("TcpBootstrap : Using%s", line); + DPRINTF("TcpBootstrap : Using%s", line); } #define TIMEOUT(__exp) \ @@ -382,7 +383,7 @@ void TcpBootstrap::Impl::establishConnections(int64_t timeoutSec) { SocketAddress nextAddr; ExtInfo info; - TRACE("establishConnections: rank %d nranks %d\n", rank_, nRanks_); + DPRINTF("establishConnections: rank %d nranks %d\n", rank_, nRanks_); auto getLeftTime = [&]() { if (connectionTimeoutUs < 0) { @@ -417,7 +418,7 @@ void TcpBootstrap::Impl::establishConnections(int64_t timeoutSec) { timespec tv; tv.tv_sec = rank / 1000; tv.tv_nsec = 1000000 * (rank % 1000); - TRACE("rank %d delaying connection to root by %ld sec %ld nsec\n", rank, + DPRINTF("rank %d delaying connection to root by %ld sec %ld nsec\n", rank, tv.tv_sec, tv.tv_nsec); (void)nanosleep(&tv, NULL); }; @@ -455,7 +456,7 @@ void TcpBootstrap::Impl::establishConnections(int64_t timeoutSec) { peerCommAddresses_[rank_] = listenSock_->getAddr(); allGather(peerCommAddresses_.data(), sizeof(SocketAddress)); - TRACE("rank %d nranks %d - DONE\n", rank_, nRanks_); + DPRINTF("rank %d nranks %d - DONE\n", rank_, nRanks_); } int TcpBootstrap::Impl::getNranksPerNode() { @@ -485,7 +486,7 @@ void TcpBootstrap::Impl::allGather(void* allData, int size) { int rank = rank_; int nRanks = nRanks_; - TRACE("allGather: rank %d nranks %d size %d\n", rank, nRanks, size); + DPRINTF("allGather: rank %d nranks %d size %d\n", rank, nRanks, size); /* Simple ring based AllGather * At each step i receive data from (rank-i-1) from left @@ -501,7 +502,7 @@ void TcpBootstrap::Impl::allGather(void* allData, int size) { netRecv(ringRecvSocket_.get(), data + rSlice * size, size); } - TRACE("allGather: rank %d nranks %d size %d - DONE\n", rank, nRanks, size); + DPRINTF("allGather: rank %d nranks %d size %d - DONE\n", rank, nRanks, size); } std::shared_ptr TcpBootstrap::Impl::getPeerSendSocket(int peer, int tag) { diff --git a/src/bootstrap/env.cpp b/src/bootstrap/env.cpp deleted file mode 100644 index f38db72523..0000000000 --- a/src/bootstrap/env.cpp +++ /dev/null @@ -1,95 +0,0 @@ -/****************************************************************************** - * Copyright (c) Microsoft Corporation. - * Modifications Copyright (c) Advanced Micro Devices, Inc. All rights reserved. - * - * SPDX-License-Identifier: MIT - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to - * deal in the Software without restriction, including without limitation the - * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or - * sell copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - *****************************************************************************/ - -#include -#include - -#include "env.hpp" -#include "utils.hpp" - -template -T readEnv(const std::string &envName, const T &defaultValue) { - const char *envCstr = getenv(envName.c_str()); - if (envCstr == nullptr) return defaultValue; - if constexpr (std::is_same_v) { - return atoi(envCstr); - } else if constexpr (std::is_same_v) { - return (std::string(envCstr) != "0"); - } - return T(envCstr); -} - -template -void readAndSetEnv(const std::string &envName, T &env) { - const char *envCstr = getenv(envName.c_str()); - if (envCstr == nullptr) return; - if constexpr (std::is_same_v) { - env = atoi(envCstr); - } else if constexpr (std::is_same_v) { - env = (std::string(envCstr) != "0"); - } else { - env = std::string(envCstr); - } -} - -template -void logEnv(const std::string &envName, const T &env) { - if (!getenv(envName.c_str())) return; - INFO("%s=%d", envName.c_str(), env); -} - -template <> -void logEnv(const std::string &envName, const std::string &env) { - if (!getenv(envName.c_str())) return; - INFO("%s=%s", envName.c_str(), env.c_str()); -} - -namespace rocshmem { - -Env::Env() - : debug(readEnv("ROCSHMEM_DEBUG", "")), - debugSubsys(readEnv("ROCSHMEM_DEBUG_SUBSYS", "")), - debugFile(readEnv("ROCSHMEM_DEBUG_FILE", "")), - hostid(readEnv("ROCSHMEM_HOSTID", "")), - socketFamily(readEnv("ROCSHMEM_SOCKET_FAMILY", "")), - socketIfname(readEnv("ROCSHMEM_SOCKET_IFNAME", "")) {} - -std::shared_ptr env() { - static std::shared_ptr globalEnv = std::shared_ptr(new Env()); - static bool logged = false; - if (!logged) { - logged = true; - // cannot log inside the constructor because of circular dependency - logEnv("ROCSHMEM_DEBUG", globalEnv->debug); - logEnv("ROCSHMEM_DEBUG_SUBSYS", globalEnv->debugSubsys); - logEnv("ROCSHMEM_DEBUG_FILE", globalEnv->debugFile); - logEnv("ROCSHMEM_HOSTID", globalEnv->hostid); - logEnv("ROCSHMEM_SOCKET_FAMILY", globalEnv->socketFamily); - logEnv("ROCSHMEM_SOCKET_IFNAME", globalEnv->socketIfname); - } - return globalEnv; -} - -} // namespace rocshmem diff --git a/src/bootstrap/env.hpp b/src/bootstrap/env.hpp deleted file mode 100644 index f5eda05730..0000000000 --- a/src/bootstrap/env.hpp +++ /dev/null @@ -1,59 +0,0 @@ -/****************************************************************************** - * Copyright (c) Microsoft Corporation. - * Modifications Copyright (c) Advanced Micro Devices, Inc. All rights reserved. - * - * SPDX-License-Identifier: MIT - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to - * deal in the Software without restriction, including without limitation the - * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or - * sell copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - *****************************************************************************/ - -#ifndef ROCSHMEM_ENV_HPP_ -#define ROCSHMEM_ENV_HPP_ - -#include -#include - -namespace rocshmem { - -class Env; - -/// Get the environment. -/// @return A reference to the global environment object. -std::shared_ptr env(); - -/// The constructor reads environment variables and sets the corresponding fields. -/// Use the @ref env() function to get the environment object. -class Env { - public: - const std::string debug; - const std::string debugSubsys; - const std::string debugFile; - const std::string hostid; - const std::string socketFamily; - const std::string socketIfname; - - private: - Env(); - - friend std::shared_ptr env(); -}; - -} // namespace rocshmem - -#endif // ROCSHMEM_ENV_HPP_ diff --git a/src/bootstrap/socket.cpp b/src/bootstrap/socket.cpp index 745b1d3bcd..5b1b57dfda 100644 --- a/src/bootstrap/socket.cpp +++ b/src/bootstrap/socket.cpp @@ -32,10 +32,11 @@ #include #include +#include #include "socket.hpp" #include "utils.hpp" -#include "env.hpp" +#include "../util.hpp" namespace rocshmem { @@ -85,7 +86,7 @@ static uint16_t socketToPort(union SocketAddress* addr) { /* Allow the user to force the IPv4/IPv6 interface selection */ static int envSocketFamily(void) { int family = -1; // Family selection is not forced, will use first one found - const std::string& socketFamily = env()->socketFamily; + const std::string& socketFamily = rocshmem_env_.get_bootstrap_socket_family(); if (socketFamily == "") return family; if (socketFamily == "AF_INET") @@ -97,7 +98,7 @@ static int envSocketFamily(void) { static int findInterfaces(const char* prefixList, char* names, union SocketAddress* addrs, int sock_family, int maxIfNameSize, int maxIfs) { -#ifdef ROCSHMEM_ENABLE_TRACE +#ifdef DEBUG char line[SOCKET_NAME_MAXLEN + 1]; #endif struct netIf userIfs[MAX_IFS]; @@ -117,7 +118,7 @@ static int findInterfaces(const char* prefixList, char* names, union SocketAddre int family = interface->ifa_addr->sa_family; if (family != AF_INET && family != AF_INET6) continue; - TRACE("Found interface %s:%s\n", interface->ifa_name, + DPRINTF("Found interface %s:%s\n", interface->ifa_name, SocketToString((union SocketAddress*)interface->ifa_addr, line)); /* Allow the caller to force the socket family type */ @@ -149,7 +150,7 @@ static int findInterfaces(const char* prefixList, char* names, union SocketAddre strncpy(names + found * maxIfNameSize, interface->ifa_name, maxIfNameSize); // Store the IP address int salen = (family == AF_INET) ? sizeof(struct sockaddr_in) : sizeof(struct sockaddr_in6); - memcpy(addrs + found, interface->ifa_addr, salen); + std::memcpy(addrs + found, interface->ifa_addr, salen); found++; } } @@ -203,7 +204,7 @@ static bool matchSubnet(struct ifaddrs local_if, union SocketAddress* remote) { int FindInterfaceMatchSubnet(char* ifNames, union SocketAddress* localAddrs, union SocketAddress* remoteAddr, int ifNameMaxSize, int maxIfs) { -#ifdef ROCSHMEM_ENABLE_TRACE +#ifdef DEBUG char line[SOCKET_NAME_MAXLEN + 1]; #endif char line_a[SOCKET_NAME_MAXLEN + 1]; @@ -224,12 +225,12 @@ int FindInterfaceMatchSubnet(char* ifNames, union SocketAddress* localAddrs, uni // Store the local IP address int salen = (family == AF_INET) ? sizeof(struct sockaddr_in) : sizeof(struct sockaddr_in6); - memcpy(localAddrs + found, interface->ifa_addr, salen); + std::memcpy(localAddrs + found, interface->ifa_addr, salen); // Store the interface name strncpy(ifNames + found * ifNameMaxSize, interface->ifa_name, ifNameMaxSize); - TRACE("NET : Found interface %s:%s in the same subnet as remote address %s\n", + DPRINTF("NET : Found interface %s:%s in the same subnet as remote address %s\n", interface->ifa_name, SocketToString(localAddrs + found, line), SocketToString(remoteAddr, line_a)); found++; if (found == maxIfs) break; @@ -273,13 +274,13 @@ void SocketGetAddrFromString(union SocketAddress* ua, const char* ip_port_pair) // use the first if (p->ai_family == AF_INET) { struct sockaddr_in& sin = ua->sin; - memcpy(&sin, p->ai_addr, sizeof(struct sockaddr_in)); + std::memcpy(&sin, p->ai_addr, sizeof(struct sockaddr_in)); sin.sin_family = AF_INET; // IPv4 // inet_pton(AF_INET, ni.prefix, &(sin.sin_addr)); // IP address sin.sin_port = htons(ni.port); // port } else if (p->ai_family == AF_INET6) { struct sockaddr_in6& sin6 = ua->sin6; - memcpy(&sin6, p->ai_addr, sizeof(struct sockaddr_in6)); + std::memcpy(&sin6, p->ai_addr, sizeof(struct sockaddr_in6)); sin6.sin6_family = AF_INET6; // IPv6 sin6.sin6_port = htons(ni.port); // port sin6.sin6_flowinfo = 0; // needed by IPv6, but possibly obsolete @@ -333,13 +334,13 @@ int FindInterfaces(char* ifNames, union SocketAddress* ifAddrs, int ifNameMaxSiz int sock_family = envSocketFamily(); // User specified interface - const std::string& socketIfname = env()->socketIfname; + const std::string& socketIfname = rocshmem_env_.get_bootstrap_socket_ifname(); if (inputIfName) { - TRACE("using iterface %s", inputIfName); + DPRINTF("using iterface %s", inputIfName); nIfs = findInterfaces(inputIfName, ifNames, ifAddrs, sock_family, ifNameMaxSize, maxIfs); } else if (socketIfname != "") { // Specified by user : find or fail - if (shownIfName++ == 0) TRACE ("ROCSHMEM_SOCKET_IFNAME set to %s", socketIfname.c_str()); + if (shownIfName++ == 0) DPRINTF ("ROCSHMEM_SOCKET_IFNAME set to %s", socketIfname.c_str()); nIfs = findInterfaces(socketIfname.c_str(), ifNames, ifAddrs, sock_family, ifNameMaxSize, maxIfs); } else { @@ -371,7 +372,7 @@ Socket::Socket(const SocketAddress* addr, uint64_t magic, enum SocketType type, if (addr) { /* IPv4/IPv6 support */ int family; - memcpy(&addr_, addr, sizeof(union SocketAddress)); + std::memcpy(&addr_, addr, sizeof(union SocketAddress)); family = addr_.sa.sa_family; if (family != AF_INET && family != AF_INET6) { char line[SOCKET_NAME_MAXLEN + 1]; @@ -441,7 +442,7 @@ void Socket::bind() { return; } if (remainSecs > 0) { - TRACE("No available ephemeral ports found, will retry after 1 second"); + DPRINTF("No available ephemeral ports found, will retry after 1 second"); sleep(1); remainSecs--; } else { @@ -460,11 +461,11 @@ void Socket::bind() { } void Socket::bindAndListen() { -#ifdef ROCSHMEM_ENABLE_TRACE +#ifdef DEBUG char line[SOCKET_NAME_MAXLEN + 1]; #endif bind(); - TRACE("Listening on socket %s\n", SocketToString(&addr_, line)); + DPRINTF("Listening on socket %s\n", SocketToString(&addr_, line)); /* Put the socket in listen mode * NB: The backlog will be silently truncated to the value in /proc/sys/net/core/somaxconn @@ -477,7 +478,7 @@ void Socket::bindAndListen() { } void Socket::connect(int64_t timeout) { -#ifdef ROCSHMEM_ENABLE_TRACE +#ifdef DEBUG char line[SOCKET_NAME_MAXLEN + 1]; #endif Timer timer; @@ -492,10 +493,10 @@ void Socket::connect(int64_t timeout) { ERROR("wrong socket state %d\n", state_); return; } - TRACE("Connecting to socket %s \n", SocketToString(&addr_, line)); + DPRINTF("Connecting to socket %s \n", SocketToString(&addr_, line)); if (setsockopt(fd_, IPPROTO_TCP, TCP_NODELAY, (char*)&one, sizeof(int)) != 0) { - INFO("setsockopt(TCP_NODELAY) failed, errno %d\n", errno); + DPRINTF("setsockopt(TCP_NODELAY) failed, errno %d\n", errno); return; } @@ -642,7 +643,7 @@ void Socket::tryAccept() { } else { usleep(SLEEP_INT); if (++acceptRetries_ % 1000 == 0) - INFO("tryAccept: Call to try accept returned %s, retrying", strerror(errno)); + DPRINTF("tryAccept: Call to try accept returned %s, retrying", strerror(errno)); } } @@ -686,7 +687,7 @@ void Socket::startConnect() { return; } else if (errno == ECONNREFUSED || errno == ETIMEDOUT) { usleep(SLEEP_INT); - if (++connectRetries_ % 1000 == 0) INFO("Call to connect returned %s, retrying", strerror(errno)); + if (++connectRetries_ % 1000 == 0) DPRINTF("Call to connect returned %s, retrying", strerror(errno)); return; } else { char line[SOCKET_NAME_MAXLEN + 1]; @@ -725,7 +726,7 @@ void Socket::pollConnect() { state_ = SocketStateConnected; } else if (ret == ECONNREFUSED || ret == ETIMEDOUT) { if (++connectRetries_ % 1000 == 0) { - INFO("Call to connect returned %s, retrying", strerror(errno)); + DPRINTF("Call to connect returned %s, retrying", strerror(errno)); } usleep(SLEEP_INT); diff --git a/src/bootstrap/utils.cpp b/src/bootstrap/utils.cpp index 22eb712e6d..829599134a 100644 --- a/src/bootstrap/utils.cpp +++ b/src/bootstrap/utils.cpp @@ -34,7 +34,7 @@ #include #include "utils.hpp" -#include "env.hpp" +#include "../util.hpp" constexpr char HOSTID_FILE[32] = "/proc/sys/kernel/random/boot_id"; @@ -101,7 +101,7 @@ uint64_t computeHostHash(void) { std::string hostName = getHostName(hashLen, '\0'); strncpy(hostHash, hostName.c_str(), hostName.size()); - std::string hostid = env()->hostid; + std::string hostid = rocshmem_env_.get_bootstrap_hostid(); if (hostid != "") { strncpy(hostHash, hostid.c_str(), hashLen); } else if (hostName.size() < hashLen) { @@ -113,7 +113,7 @@ uint64_t computeHostHash(void) { // Make sure the string is terminated hostHash[sizeof(hostHash) - 1] = '\0'; - TRACE("unique hostname '%s'", hostHash); + DPRINTF("unique hostname '%s'", hostHash); return getHash(hostHash, strlen(hostHash)); } @@ -141,7 +141,7 @@ uint64_t computePidHash(void) { if (len < 0) len = 0; pname[plen + len] = '\0'; - TRACE("unique PID '%s'", pname); + DPRINTF("unique PID '%s'", pname); return getHash(pname, strlen(pname)); } diff --git a/src/bootstrap/utils.hpp b/src/bootstrap/utils.hpp index 056499b900..ed720b9d6d 100644 --- a/src/bootstrap/utils.hpp +++ b/src/bootstrap/utils.hpp @@ -33,18 +33,6 @@ #define ERROR(...) { fprintf(stderr, __VA_ARGS__); abort(); } -#ifdef ROCSHMEM_ENABLE_TRACE -#define TRACE(...) printf(__VA_ARGS__) -#else -#define TRACE(...) -#endif - -#if defined ROCSHMEM_ENABLE_INFO -#define INFO(FLAGS, ...)printf(__VA_ARGS__) -#else -#define INFO(...) -#endif - namespace rocshmem { struct Timer { diff --git a/src/ipc_policy.cpp b/src/ipc_policy.cpp index eee40e728c..ada1609f28 100644 --- a/src/ipc_policy.cpp +++ b/src/ipc_policy.cpp @@ -109,7 +109,7 @@ __host__ void IpcOnImpl::ipcHostInit(int my_pe, const HEAP_BASES_T &heap_bases, */ free(vec_ipc_handle); - if (0 == rocshmem_env_config.ro_disable_ipc) { + if (0 == rocshmem_env_.get_ro_disable_ipc()) { int thread_comm_rank; CHECK_HIP(hipMalloc(reinterpret_cast(&pes_with_ipc_avail), shm_size * sizeof(int))); diff --git a/src/reverse_offload/mpi_transport.cpp b/src/reverse_offload/mpi_transport.cpp index a520a176ea..b7342cb686 100644 --- a/src/reverse_offload/mpi_transport.cpp +++ b/src/reverse_offload/mpi_transport.cpp @@ -590,6 +590,8 @@ std::unique_ptr MPITransport::raw_requests() { } void MPITransport::progress() { + static int progress_delay = rocshmem_env_.get_ro_progress_delay(); + if (requests.size() == 0) { const int tag{1000}; int flag{0}; @@ -597,7 +599,7 @@ void MPITransport::progress() { // Slowing the progress engine down a bit avoid hammering the memory subsystem. // This leads to significant performance benefits - usleep (rocshmem_env_config.ro_progress_delay); + usleep (progress_delay); NET_CHECK(MPI_Iprobe(MPI_ANY_SOURCE, tag, ro_net_comm_world, &flag, &status)); } else { DPRINTF("Testing all outstanding requests (%zu)\n", requests.size()); diff --git a/src/rocshmem.cpp b/src/rocshmem.cpp index 4b5b5765ba..dd49b349eb 100644 --- a/src/rocshmem.cpp +++ b/src/rocshmem.cpp @@ -86,8 +86,6 @@ rocshmem_ctx_t ROCSHMEM_HOST_CTX_DEFAULT; rocm_init(); - rocshmem_env_config_init(); - #ifdef USE_RO CHECK_HIP(hipHostMalloc(&backend, sizeof(ROBackend))); backend = new (backend) ROBackend(comm); @@ -155,14 +153,7 @@ rocshmem_ctx_t ROCSHMEM_HOST_CTX_DEFAULT; TcpBootstrap bootstr(attr->rank, attr->nranks); - int timeout = 5; - char *value; - value = getenv("ROCSHMEM_BOOTSTRAP_TIMEOUT"); - if (value != nullptr) { - timeout = atoi(value); - } - - bootstr.initialize(attr->uid, timeout); + bootstr.initialize(attr->uid, rocshmem_env_.get_bootstrap_timeout()); int *inc_ranks = new int[attr->nranks]; inc_ranks[attr->rank] = world_rank; diff --git a/src/util.cpp b/src/util.cpp index c1845a3b0c..76ff42aa4f 100644 --- a/src/util.cpp +++ b/src/util.cpp @@ -34,6 +34,8 @@ namespace rocshmem { __constant__ int* print_lock; +rocshmem_env_config rocshmem_env_; + typedef struct device_agent { hsa_agent_t agent; hsa_amd_memory_pool_t pool; @@ -121,21 +123,63 @@ void rocm_memory_lock_to_fine_grain(void* ptr, size_t size, void** gpu_ptr, } } -struct rocshmem_env_config_t rocshmem_env_config; -void rocshmem_env_config_init(void) { +rocshmem_env_config::rocshmem_env_config() { char* env_value = NULL; env_value = getenv("ROCSHMEM_RO_DISABLE_IPC"); if (NULL != env_value) { - rocshmem_env_config.ro_disable_ipc = atoi(env_value); + ro_disable_ipc = atoi(env_value); } env_value = getenv("ROCSHMEM_RO_PROGRESS_DELAY"); if (nullptr != env_value) { - rocshmem_env_config.ro_progress_delay = atoi(env_value); + ro_progress_delay = atoi(env_value); } + env_value = getenv("ROCSHMEM_BOOTSTRAP_TIMEOUT"); + if (nullptr != env_value) { + bootstrap_timeout = atoi(env_value); + } + + env_value = getenv("ROCSHMEM_BOOTSTRAP_HOSTID"); + if (nullptr != env_value) { + bootstrap_hostid = std::string(env_value); + } + + env_value = getenv("ROCSHMEM_BOOTSTRAP_SOCKET_FAMILY"); + if (nullptr != env_value) { + bootstrap_socket_family = std::string(env_value); + } + + env_value = getenv("ROCSHMEM_BOOTSTRAP_SOCKET_IFNAME"); + if (nullptr != env_value) { + bootstrap_socket_ifname = std::string(env_value); + } +} + +int rocshmem_env_config::get_ro_disable_ipc() { + return ro_disable_ipc; +} + +int rocshmem_env_config::get_ro_progress_delay() { + return ro_progress_delay; +} + +int rocshmem_env_config::get_bootstrap_timeout() { + return bootstrap_timeout; +} + +std::string rocshmem_env_config::get_bootstrap_hostid() { + return bootstrap_hostid; +} + +std::string rocshmem_env_config::get_bootstrap_socket_family() { + return bootstrap_socket_family; +} + +std::string rocshmem_env_config::get_bootstrap_socket_ifname() { + return bootstrap_socket_ifname; } } // namespace rocshmem diff --git a/src/util.hpp b/src/util.hpp index 3c2e48ec41..3f9a6115a9 100644 --- a/src/util.hpp +++ b/src/util.hpp @@ -268,13 +268,27 @@ int rocm_init(); void rocm_memory_lock_to_fine_grain(void* ptr, size_t size, void** gpu_ptr, int gpu_id); -struct rocshmem_env_config_t { +class rocshmem_env_config { +public: + rocshmem_env_config(); + + int get_ro_disable_ipc(); + int get_ro_progress_delay(); + int get_bootstrap_timeout(); + std::string get_bootstrap_hostid(); + std::string get_bootstrap_socket_family(); + std::string get_bootstrap_socket_ifname(); + +private: int ro_disable_ipc = 0; int ro_progress_delay = 3; + int bootstrap_timeout = 5; + std::string bootstrap_hostid; + std::string bootstrap_socket_family; + std::string bootstrap_socket_ifname; }; -extern struct rocshmem_env_config_t rocshmem_env_config; -void rocshmem_env_config_init(void); +extern rocshmem_env_config rocshmem_env_; } // namespace rocshmem