Add MSCCL Support (#658)
* Add MSCCL support * Add alignment and message size checking * Fix nRanks checking, in-place and out-of-place tests and group call handling * Fix hipGraph unit test * Change MSCCL init warning to INFO * Revise license info
This commit is contained in:
Ziyue Yang
GitHub
+1
-5
@@ -1,5 +1,6 @@
|
||||
/*************************************************************************
|
||||
* Copyright (c) 2015-2017, NVIDIA CORPORATION. All rights reserved.
|
||||
* Modifications Copyright (c) Microsoft Corporation. Licensed under the MIT License.
|
||||
*
|
||||
* See LICENSE.txt for license information
|
||||
************************************************************************/
|
||||
@@ -67,11 +68,6 @@ extern __thread struct ncclComm* ncclGroupCommHead;
|
||||
extern __thread struct ncclComm* ncclGroupCommPreconnectHead;
|
||||
extern __thread int ncclGroupBlocking;
|
||||
|
||||
inline ncclResult_t ncclGroupStartInternal() {
|
||||
ncclGroupDepth++;
|
||||
return ncclSuccess;
|
||||
}
|
||||
|
||||
inline ncclResult_t ncclGroupErrCheck(ncclResult_t ret) {
|
||||
if (ncclGroupDepth > 0) {
|
||||
if (ret != ncclSuccess && ret != ncclInProgress) ncclGroupError = ret;
|
||||
|
||||
@@ -0,0 +1,49 @@
|
||||
/*************************************************************************
|
||||
* Copyright (c) Microsoft Corporation.
|
||||
* Licensed under the MIT License.
|
||||
************************************************************************/
|
||||
|
||||
#ifndef MSCCL_KERNEL_H_
|
||||
#define MSCCL_KERNEL_H_
|
||||
|
||||
#define MSCCL_KERNEL_ENTRY_NAME(devredop, type, proto) mscclKernel_##devredop##_##type##_##proto
|
||||
|
||||
#define MSCCL_DECL_KERNEL_ENTRY_FUNC_DEVREDOP_TYPE_PROTO(devredop, type, proto) \
|
||||
__global__ void MSCCL_KERNEL_ENTRY_NAME(devredop, type, proto)(struct ncclDevComm* comm, struct mscclAlgo* algo, struct mscclWork work);
|
||||
|
||||
#define MSCCL_DECL_KERNEL_ENTRY_FUNC_DEVREDOP_TYPE(devredop, type) \
|
||||
MSCCL_DECL_KERNEL_ENTRY_FUNC_DEVREDOP_TYPE_PROTO(devredop, type, LL) \
|
||||
MSCCL_DECL_KERNEL_ENTRY_FUNC_DEVREDOP_TYPE_PROTO(devredop, type, LL128) \
|
||||
MSCCL_DECL_KERNEL_ENTRY_FUNC_DEVREDOP_TYPE_PROTO(devredop, type, Simple)
|
||||
|
||||
#define MSCCL_DECL_KERNEL_ENTRY_FUNC_DEVREDOP(devredop) \
|
||||
MSCCL_DECL_KERNEL_ENTRY_FUNC_DEVREDOP_TYPE(devredop, int8_t) \
|
||||
MSCCL_DECL_KERNEL_ENTRY_FUNC_DEVREDOP_TYPE(devredop, uint8_t) \
|
||||
MSCCL_DECL_KERNEL_ENTRY_FUNC_DEVREDOP_TYPE(devredop, int32_t) \
|
||||
MSCCL_DECL_KERNEL_ENTRY_FUNC_DEVREDOP_TYPE(devredop, uint32_t) \
|
||||
MSCCL_DECL_KERNEL_ENTRY_FUNC_DEVREDOP_TYPE(devredop, int64_t) \
|
||||
MSCCL_DECL_KERNEL_ENTRY_FUNC_DEVREDOP_TYPE(devredop, uint64_t) \
|
||||
MSCCL_DECL_KERNEL_ENTRY_FUNC_DEVREDOP_TYPE(devredop, half) \
|
||||
MSCCL_DECL_KERNEL_ENTRY_FUNC_DEVREDOP_TYPE(devredop, float) \
|
||||
MSCCL_DECL_KERNEL_ENTRY_FUNC_DEVREDOP_TYPE(devredop, double) \
|
||||
MSCCL_DECL_KERNEL_ENTRY_FUNC_DEVREDOP_TYPE(devredop, rccl_bfloat16)
|
||||
|
||||
#define MSCCL_DECL_KERNEL_ENTRY_FUNC_DEVREDOP_NOFLOAT(devredop) \
|
||||
MSCCL_DECL_KERNEL_ENTRY_FUNC_DEVREDOP_TYPE(devredop, int8_t) \
|
||||
MSCCL_DECL_KERNEL_ENTRY_FUNC_DEVREDOP_TYPE(devredop, uint8_t) \
|
||||
MSCCL_DECL_KERNEL_ENTRY_FUNC_DEVREDOP_TYPE(devredop, int32_t) \
|
||||
MSCCL_DECL_KERNEL_ENTRY_FUNC_DEVREDOP_TYPE(devredop, uint32_t) \
|
||||
MSCCL_DECL_KERNEL_ENTRY_FUNC_DEVREDOP_TYPE(devredop, int64_t) \
|
||||
MSCCL_DECL_KERNEL_ENTRY_FUNC_DEVREDOP_TYPE(devredop, uint64_t)
|
||||
|
||||
#define MSCCL_DECL_KERNEL_ENTRY_FUNC() \
|
||||
MSCCL_DECL_KERNEL_ENTRY_FUNC_DEVREDOP(Sum) \
|
||||
MSCCL_DECL_KERNEL_ENTRY_FUNC_DEVREDOP(Prod) \
|
||||
MSCCL_DECL_KERNEL_ENTRY_FUNC_DEVREDOP(Min) \
|
||||
MSCCL_DECL_KERNEL_ENTRY_FUNC_DEVREDOP(Max) \
|
||||
MSCCL_DECL_KERNEL_ENTRY_FUNC_DEVREDOP(PreMulSum) \
|
||||
MSCCL_DECL_KERNEL_ENTRY_FUNC_DEVREDOP_NOFLOAT(SumPostDiv)
|
||||
|
||||
MSCCL_DECL_KERNEL_ENTRY_FUNC()
|
||||
|
||||
#endif
|
||||
@@ -0,0 +1,35 @@
|
||||
/*************************************************************************
|
||||
* Copyright (c) Microsoft Corporation.
|
||||
* Licensed under the MIT License.
|
||||
************************************************************************/
|
||||
|
||||
#ifndef MSCCL_LIFECYCLE_H_
|
||||
#define MSCCL_LIFECYCLE_H_
|
||||
|
||||
#include "enqueue.h"
|
||||
|
||||
#include "msccl/msccl_struct.h"
|
||||
|
||||
bool mscclEnabled();
|
||||
|
||||
void mscclSetIsCallerFlag();
|
||||
void mscclClearIsCallerFlag();
|
||||
bool mscclIsCaller();
|
||||
|
||||
bool mscclAvailable();
|
||||
|
||||
ncclResult_t mscclInit(ncclComm_t comm);
|
||||
|
||||
ncclResult_t mscclGroupStart();
|
||||
|
||||
ncclResult_t mscclEnqueueCheck(
|
||||
const void* sendbuff, const size_t sendcounts[], const size_t sdispls[],
|
||||
void* recvbuff, const size_t recvcounts[], const size_t rdispls[],
|
||||
size_t count, ncclDataType_t datatype, int root, int peer, ncclRedOp_t op,
|
||||
mscclFunc_t mscclFunc, ncclComm_t comm, hipStream_t stream);
|
||||
|
||||
ncclResult_t mscclGroupEnd();
|
||||
|
||||
ncclResult_t mscclTeardown();
|
||||
|
||||
#endif
|
||||
@@ -0,0 +1,103 @@
|
||||
/*************************************************************************
|
||||
* Copyright (c) 2019-2022, NVIDIA CORPORATION. All rights reserved.
|
||||
* Modifications Copyright (c) 2019-2022 Advanced Micro Devices, Inc. All rights reserved.
|
||||
* Modifications Copyright (c) Microsoft Corporation. Licensed under the MIT License.
|
||||
*
|
||||
* See LICENSE.txt for license information
|
||||
************************************************************************/
|
||||
|
||||
#ifndef MSCCL_PARSER_H_
|
||||
#define MSCCL_PARSER_H_
|
||||
|
||||
#include "nccl.h"
|
||||
#include "debug.h"
|
||||
#include "checks.h"
|
||||
#include <stdlib.h>
|
||||
|
||||
#include "msccl/msccl_struct.h"
|
||||
|
||||
// A few constraints to make the implementation easy
|
||||
#define MAX_STR_LEN 255
|
||||
#define MAX_ATTR_COUNT 16
|
||||
#define MAX_SUBS 1024
|
||||
#define MAX_NODES 4096
|
||||
|
||||
#define NODE_TYPE_NONE 0
|
||||
#define NODE_TYPE_OPEN 1
|
||||
#define NODE_TYPE_CLOSE 2
|
||||
#define NODE_TYPE_SINGLE 3
|
||||
|
||||
struct mscclXmlNode {
|
||||
char name[MAX_STR_LEN+1];
|
||||
struct {
|
||||
char key[MAX_STR_LEN+1];
|
||||
char value[MAX_STR_LEN+1];
|
||||
} attrs[MAX_ATTR_COUNT+1]; // Need an extra one to consume extra params
|
||||
int nAttrs;
|
||||
int type;
|
||||
struct mscclXmlNode* parent;
|
||||
struct mscclXmlNode* subs[MAX_SUBS];
|
||||
int nSubs;
|
||||
};
|
||||
|
||||
struct mscclXml {
|
||||
struct mscclXmlNode nodes[MAX_NODES];
|
||||
int maxIndex;
|
||||
};
|
||||
|
||||
static ncclResult_t mscclXmlGetAttrIndex(struct mscclXmlNode* node, const char* attrName, int* index) {
|
||||
*index = -1;
|
||||
const int nAttrs = node->nAttrs;
|
||||
for (int a=0; a<nAttrs; a++) {
|
||||
if (strncmp(node->attrs[a].key, attrName, MAX_STR_LEN) == 0) {
|
||||
*index = a;
|
||||
return ncclSuccess;
|
||||
}
|
||||
}
|
||||
return ncclSuccess;
|
||||
}
|
||||
|
||||
static ncclResult_t mscclXmlGetAttr(struct mscclXmlNode* node, const char* attrName, const char** value) {
|
||||
int index;
|
||||
NCCLCHECK(mscclXmlGetAttrIndex(node, attrName, &index));
|
||||
*value = index == -1 ? NULL : node->attrs[index].value;
|
||||
return ncclSuccess;
|
||||
}
|
||||
|
||||
static ncclResult_t mscclXmlGetAttrStr(struct mscclXmlNode* node, const char* attrName, const char** value) {
|
||||
NCCLCHECK(mscclXmlGetAttr(node, attrName, value));
|
||||
if (*value == NULL) {
|
||||
WARN("Attribute %s of node %s not found", attrName, node->name);
|
||||
return ncclInternalError;
|
||||
}
|
||||
return ncclSuccess;
|
||||
}
|
||||
static ncclResult_t mscclXmlGetAttrInt(struct mscclXmlNode* node, const char* attrName, int* value) {
|
||||
const char* str;
|
||||
NCCLCHECK(mscclXmlGetAttrStr(node, attrName, &str));
|
||||
*value = strtol(str, NULL, 0);
|
||||
return ncclSuccess;
|
||||
}
|
||||
|
||||
static ncclResult_t mscclXmlGetAttrInt64(struct mscclXmlNode* node, const char* attrName, int64_t* value) {
|
||||
const char* str;
|
||||
NCCLCHECK(mscclXmlGetAttrStr(node, attrName, &str));
|
||||
*value = strtoll(str, NULL, 0);
|
||||
return ncclSuccess;
|
||||
}
|
||||
|
||||
static ncclResult_t mscclXmlFindTag(struct mscclXml* xml, const char* tagName, struct mscclXmlNode** node) {
|
||||
*node = NULL;
|
||||
for (int i=0; i<xml->maxIndex; i++) {
|
||||
struct mscclXmlNode* n = xml->nodes+i;
|
||||
if (strcmp(n->name, tagName) == 0) {
|
||||
*node = n;
|
||||
return ncclSuccess;
|
||||
}
|
||||
}
|
||||
return ncclSuccess;
|
||||
}
|
||||
|
||||
ncclResult_t mscclGetAlgoFromXmlFile(const char* xmlGraphFile, struct mscclAlgo* algo, int rank);
|
||||
|
||||
#endif
|
||||
@@ -0,0 +1,28 @@
|
||||
/*************************************************************************
|
||||
* Copyright (c) Microsoft Corporation.
|
||||
* Licensed under the MIT License.
|
||||
************************************************************************/
|
||||
|
||||
#ifndef MSCCL_SETUP_H_
|
||||
#define MSCCL_SETUP_H_
|
||||
|
||||
#include <hip/hip_runtime.h>
|
||||
|
||||
#include "comm.h"
|
||||
#include "msccl/msccl_struct.h"
|
||||
|
||||
ncclResult_t mscclSetupScratch(struct mscclAlgo* hostAlgo, hipStream_t stream);
|
||||
|
||||
ncclResult_t mscclSetupSyncFlags(hipStream_t stream);
|
||||
|
||||
ncclResult_t mscclSetupConnections(struct mscclAlgo* hostAlgo, ncclComm_t comm);
|
||||
|
||||
ncclResult_t mscclSetupCount(struct mscclAlgo* hostAlgo, ncclComm_t comm, size_t count, ncclDataType_t dataType);
|
||||
|
||||
ncclResult_t mscclSetupProxy(struct mscclAlgo* hostAlgo, ncclComm_t comm);
|
||||
|
||||
ncclResult_t mscclSetupKernel(const void* sendBuff, void* recvBuff, size_t count,
|
||||
ncclDataType_t dataType, ncclRedOp_t op, struct mscclAlgo* hostAlgo, struct mscclAlgo* devAlgo,
|
||||
ncclComm_t comm, hipStream_t stream);
|
||||
|
||||
#endif
|
||||
@@ -0,0 +1,13 @@
|
||||
/*************************************************************************
|
||||
* Copyright (c) Microsoft Corporation.
|
||||
* Licensed under the MIT License.
|
||||
************************************************************************/
|
||||
|
||||
#ifndef MSCCL_STATUS_H_
|
||||
#define MSCCL_STATUS_H_
|
||||
|
||||
#include "msccl/msccl_struct.h"
|
||||
|
||||
mscclStatus& mscclGetStatus();
|
||||
|
||||
#endif
|
||||
@@ -0,0 +1,209 @@
|
||||
/*************************************************************************
|
||||
* Copyright (c) Microsoft Corporation.
|
||||
* Licensed under the MIT License.
|
||||
************************************************************************/
|
||||
|
||||
#ifndef MSCCL_STRUCT_H_
|
||||
#define MSCCL_STRUCT_H_
|
||||
|
||||
#include <cstdint>
|
||||
#include <map>
|
||||
#include <set>
|
||||
#include <vector>
|
||||
#include "devcomm.h"
|
||||
|
||||
#define MSCCL_MAX_NUM_STEPS 256
|
||||
#define MSCCL_MAX_NUM_THREAD_BLOCKS_PER_CHANNEL 32
|
||||
#define MSCCL_MAX_NUM_THREAD_BLOCKS (MSCCL_MAX_NUM_THREAD_BLOCKS_PER_CHANNEL * MAXCHANNELS)
|
||||
#define MSCCL_MAX_COUNT 72 // max concurrent number of msccl chunk transmission
|
||||
#define MSCCL_MAX_REDUCE_FUSION 16
|
||||
#define MSCCL_MAX_NUM_ALGOS 1024
|
||||
|
||||
#define MSCCL_SLICESTEPS (NCCL_STEPS/4)
|
||||
#define MSCCL_CHUNKSTEPS (NCCL_STEPS/2)
|
||||
|
||||
#define MSCCL_INPUT_BUFFER 0
|
||||
#define MSCCL_OUTPUT_BUFFER 1
|
||||
#define MSCCL_SCRATCH_BUFFER 2
|
||||
|
||||
#define MSCCL_SEND 0
|
||||
#define MSCCL_RECV 1
|
||||
#define MSCCL_RECV_COPY_SEND 2
|
||||
#define MSCCL_RECV_REDUCE_SEND 3
|
||||
#define MSCCL_RECV_REDUCE_COPY 4
|
||||
#define MSCCL_RECV_REDUCE_COPY_SEND 5
|
||||
#define MSCCL_LOCAL_COPY 6
|
||||
#define MSCCL_REDUCE 7
|
||||
|
||||
typedef enum { mscclFuncReduce = 0,
|
||||
mscclFuncBroadcast = 1,
|
||||
mscclFuncAllReduce = 2,
|
||||
mscclFuncReduceScatter = 3,
|
||||
mscclFuncAllGather = 4,
|
||||
mscclFuncSend = 5,
|
||||
mscclFuncRecv = 6,
|
||||
mscclFuncGather = 7,
|
||||
mscclFuncScatter = 8,
|
||||
mscclFuncAllToAll = 9,
|
||||
mscclFuncAllToAllv = 10,
|
||||
mscclNumFuncs = 11 } mscclFunc_t;
|
||||
|
||||
struct mscclTransmission {
|
||||
int16_t dependencePointer; // index to the first dependence
|
||||
int16_t numDependencies; // dependencePointer+numDependencies indicate the last dependence
|
||||
int16_t reductionPointer; // where the reduction starts
|
||||
int16_t numReductions; // number of reductions with the same dst
|
||||
int16_t srcOffset;
|
||||
int16_t dstOffset;
|
||||
uint8_t srcBuffer : 4; // input/output/scratch
|
||||
uint8_t dstBuffer : 4; // input/output/scratch
|
||||
int8_t hasDependence;
|
||||
uint8_t type;
|
||||
uint8_t count;
|
||||
}; // 16 bytes
|
||||
|
||||
static_assert((1ULL << (8*sizeof(mscclTransmission::count))) - 1 > MSCCL_MAX_COUNT, "MSCCL_MAX_COUNT must representable by datatype of count");
|
||||
|
||||
struct mscclThreadBlock {
|
||||
// step is used to index into these arrays
|
||||
struct mscclTransmission transmissions[MSCCL_MAX_NUM_STEPS]; // 4KB
|
||||
int8_t dependentBid[MSCCL_MAX_NUM_STEPS]; // -1 if not dependent on any thread block, 256 bytes
|
||||
int16_t dependentStep[MSCCL_MAX_NUM_STEPS]; // 512 bytes
|
||||
int16_t reductionSrcOffsets[MSCCL_MAX_NUM_STEPS]; // 512 bytes
|
||||
int16_t sendPeer;
|
||||
int16_t recvPeer;
|
||||
uint16_t nSteps;
|
||||
int16_t channelId; // associated channel. -1 indicates a thread block with only local copies
|
||||
}; // 5384 bytes
|
||||
|
||||
static_assert(sizeof(struct mscclThreadBlock) % sizeof(uint64_t) == 0, "Sanity check: sizeof(struct mscclThreadBlock) \% sizeof(uint64_t) != 0");
|
||||
|
||||
struct mscclFlag {
|
||||
uint64_t flag;
|
||||
uint64_t align[3]; // to avoid false sharing
|
||||
};
|
||||
|
||||
struct mscclChannelPeerInfo {
|
||||
int peer;
|
||||
// nTransmissionsOfCount[i]: number of transmissions with count i (in terms of msccl chunks)
|
||||
int nTransmissionsOfCount[MSCCL_MAX_COUNT + 1];
|
||||
int existingCounts[MSCCL_MAX_COUNT + 1];
|
||||
int nExistingCounts;
|
||||
};
|
||||
|
||||
struct mscclChannelInfo {
|
||||
struct mscclChannelPeerInfo sendPeerInfo[MSCCL_MAX_NUM_THREAD_BLOCKS_PER_CHANNEL];
|
||||
int nSendPeers;
|
||||
struct mscclChannelPeerInfo recvPeerInfo[MSCCL_MAX_NUM_THREAD_BLOCKS_PER_CHANNEL];
|
||||
int nRecvPeers;
|
||||
};
|
||||
|
||||
struct mscclAlgo {
|
||||
// number of chunks of input/output in each MSCCL algorithm loop
|
||||
int nChunksPerLoop;
|
||||
// the protocol that the algorithm needs to use
|
||||
int protocol;
|
||||
// number of channels needed by MSCCL algorithm
|
||||
int nChannels;
|
||||
// number of ranks required by this algorithm
|
||||
int nRanks;
|
||||
// number of necessary thread blocks
|
||||
int nBlocks;
|
||||
// number of scratch chunks that MSCCL will use
|
||||
int nScratchChunks;
|
||||
// need to times nRanks for all-gather, reduce-scatter and all-to-all
|
||||
int sizeMultiplier;
|
||||
// number of steps per chunk for this algorithm
|
||||
int chunkSteps;
|
||||
// number of steps per slice for this algorithm
|
||||
int sliceSteps;
|
||||
// bid is used as an index into this array
|
||||
struct mscclThreadBlock mscclTBs[MSCCL_MAX_NUM_THREAD_BLOCKS];
|
||||
// used to calculate proxy info
|
||||
struct mscclChannelInfo mscclChannels[MAXCHANNELS];
|
||||
// Whether the algorithm requires reduce operation
|
||||
bool hasReduce;
|
||||
// MSCCL function type
|
||||
mscclFunc_t func;
|
||||
// Min message size allowed for this algorithm.
|
||||
int64_t minBytes;
|
||||
// Max message size allowed for this algorithm, 0 for no limit.
|
||||
int64_t maxBytes;
|
||||
// Whether this algorithm is suitable for in-place.
|
||||
bool inPlace;
|
||||
// Whether this algorithm is suitable for out-of-place.
|
||||
bool outOfPlace;
|
||||
};
|
||||
|
||||
enum mscclGroupStatus {
|
||||
mscclNoGroup,
|
||||
mscclGroupSupportedOp,
|
||||
mscclGroupUnsupportedOp
|
||||
};
|
||||
|
||||
struct mscclSchedulerParam {
|
||||
const void* sendBuff;
|
||||
const size_t* sendCounts;
|
||||
std::vector<size_t> savedSendCounts;
|
||||
const size_t* sDisPls;
|
||||
std::vector<size_t> savedSDisPls;
|
||||
void* recvBuff;
|
||||
const size_t* recvCounts;
|
||||
std::vector<size_t> savedRecvCounts;
|
||||
const size_t* rDisPls;
|
||||
std::vector<size_t> savedRDisPls;
|
||||
size_t count;
|
||||
ncclDataType_t dataType;
|
||||
int root;
|
||||
int peer;
|
||||
ncclRedOp_t op;
|
||||
mscclFunc_t func;
|
||||
bool scheduled;
|
||||
mscclAlgoHandle_t handle;
|
||||
ncclComm_t comm;
|
||||
hipStream_t stream;
|
||||
};
|
||||
|
||||
struct mscclStatus {
|
||||
std::vector<mscclAlgoHandle_t> freeAlgoHandles;
|
||||
std::map<mscclAlgoHandle_t, mscclAlgo *> hostAlgos;
|
||||
std::map<mscclAlgoHandle_t, mscclAlgo *> devAlgos;
|
||||
struct mscclFlag* syncFlags;
|
||||
void *scratchBuffer;
|
||||
uint64_t scratchBufferSize;
|
||||
size_t nBytes;
|
||||
int stepSize;
|
||||
int chunkSteps;
|
||||
int sliceSteps;
|
||||
int chunkSize;
|
||||
int chunkEffectiveSize;
|
||||
int rank;
|
||||
uint32_t workIndex;
|
||||
uint32_t maxAllowedCount;
|
||||
ncclDataType_t dataType;
|
||||
mscclGroupStatus groupStatus;
|
||||
int groupDepth;
|
||||
std::vector<struct mscclSchedulerParam> savedSchedulerParams;
|
||||
};
|
||||
|
||||
struct alignas(16) mscclWork {
|
||||
volatile struct mscclFlag *syncFlags;
|
||||
void *scratchBuffer;
|
||||
const void *sendBuff;
|
||||
void *recvBuff;
|
||||
size_t count;
|
||||
uint64_t redOpArg;
|
||||
uint32_t workIndex;
|
||||
int nChunksPerLoop;
|
||||
uint32_t maxAllowedCount;
|
||||
bool hasReduce;
|
||||
bool redOpArgIsPtr;
|
||||
};
|
||||
|
||||
struct mscclShmemData {
|
||||
struct mscclThreadBlock mscclTB;
|
||||
alignas(16) struct mscclWork work;
|
||||
};
|
||||
static_assert(offsetof(struct mscclShmemData, work) % 16 == 0, "mscclShmemData.work needs to be 16B aligned");
|
||||
|
||||
#endif
|
||||
@@ -1,6 +1,7 @@
|
||||
/*************************************************************************
|
||||
* Copyright (c) 2016-2022, NVIDIA CORPORATION. All rights reserved.
|
||||
* Modifications Copyright (c) 2019-2022 Advanced Micro Devices, Inc. All rights reserved.
|
||||
* Modifications Copyright (c) Microsoft Corporation. Licensed under the MIT License.
|
||||
*
|
||||
* See LICENSE.txt for license information
|
||||
************************************************************************/
|
||||
@@ -224,4 +225,8 @@ enum ncclProxyMsgType {
|
||||
ncclResult_t ncclProxyCall(struct ncclProxyConnector* proxyConn, int type, void* reqBuff, int reqSize, void* respBuff, int respSize);
|
||||
ncclResult_t ncclProxyDestroy(struct ncclComm* comm);
|
||||
ncclResult_t ncclProxyShmUnlink(struct ncclComm* comm);
|
||||
|
||||
enum { proxyRecv=0, proxySend=1 };
|
||||
ncclResult_t mscclSaveProxy(struct ncclChannel* channel, int type, int peer, struct ncclProxyOp* op, int connIndex);
|
||||
|
||||
#endif
|
||||
|
||||
Reference in New Issue
Block a user