Ameya Keshava Mallya
42d84317cf
git-subtree-dir: projects/rccl git-subtree-mainline:3fd8a0d393git-subtree-split:1f2f9f33ba
105 lines
5.6 KiB
Plaintext
105 lines
5.6 KiB
Plaintext
# AllReduce configurations for multi-node setups - 8B-128M
|
|
# 2 nodes, 16 ranks total - 8B-128M
|
|
allreduce,8,65536,tree,ll,4,2,16,-1,-1 # Small: tree (8B to 64K)
|
|
allreduce,65537,16777216,ring,ll128,6,2,16,-1,-1 # Medium: ring (64K to 16M)
|
|
allreduce,16777217,134217728,ring,simple,8,2,16,-1,-1 # Large: ring/simple (16M to 128M)
|
|
|
|
# 3 nodes, 24 ranks total - 8B-128M
|
|
allreduce,8,65536,tree,ll,4,2,24,-1,-1 # Small: tree (8B to 64K)
|
|
allreduce,65537,16777216,ring,ll128,6,2,24,-1,-1 # Medium: ring (64K to 16M)
|
|
allreduce,16777217,134217728,ring,simple,8,2,24,-1,-1 # Large: ring/simple (16M to 128M)
|
|
|
|
# 4 nodes, 32 ranks total - 8B-128M
|
|
allreduce,8,65536,tree,ll,4,2,32,-1,-1 # Small: tree (8B to 64K)
|
|
allreduce,65537,16777216,ring,ll128,6,2,32,-1,-1 # Medium: ring (64K to 16M)
|
|
allreduce,16777217,134217728,ring,simple,8,2,32,-1,-1 # Large: ring/simple (16M to 128M)
|
|
|
|
# 5 nodes, 40 ranks total - 8B-128M
|
|
allreduce,8,65536,tree,ll,4,2,40,-1,-1 # Small: tree (8B to 64K)
|
|
allreduce,65537,16777216,ring,ll128,6,2,40,-1,-1 # Medium: ring (64K to 16M)
|
|
allreduce,16777217,134217728,ring,simple,8,2,40,-1,-1 # Large: ring/simple (16M to 128M)
|
|
|
|
# Broadcast configurations for multi-node setups - 8B-128M
|
|
# 2 nodes, 16 ranks total
|
|
broadcast,8,65536,ring,ll,4,2,16,-1,-1 # Small: ring (8B to 64K)
|
|
broadcast,65537,16777216,ring,ll128,6,2,16,-1,-1 # Medium: ring (64K to 16M)
|
|
broadcast,16777217,134217728,ring,simple,8,2,16,-1,-1 # Large: ring/simple (16M to 128M)
|
|
|
|
# 3 nodes, 24 ranks total
|
|
broadcast,8,65536,ring,ll,4,2,24,-1,-1 # Small: ring (8B to 64K)
|
|
broadcast,65537,16777216,ring,ll128,6,2,24,-1,-1 # Medium: ring (64K to 16M)
|
|
broadcast,16777217,134217728,ring,simple,8,2,24,-1,-1 # Large: ring/simple (16M to 128M)
|
|
|
|
# 4 nodes, 32 ranks total
|
|
broadcast,8,65536,ring,ll,4,2,32,-1,-1 # Small: ring (8B to 64K)
|
|
broadcast,65537,16777216,ring,ll128,6,2,32,-1,-1 # Medium: ring (64K to 16M)
|
|
broadcast,16777217,134217728,ring,simple,8,2,32,-1,-1 # Large: ring/simple (16M to 128M)
|
|
|
|
# 5 nodes, 40 ranks total
|
|
broadcast,8,65536,ring,ll,4,2,40,-1,-1 # Small: ring (8B to 64K)
|
|
broadcast,65537,16777216,ring,ll128,6,2,40,-1,-1 # Medium: ring (64K to 16M)
|
|
broadcast,16777217,134217728,ring,simple,8,2,40,-1,-1 # Large: ring/simple (16M to 128M)
|
|
|
|
# AllGather configurations for multi-node setups - 8B-128M
|
|
# 2 nodes, 16 ranks total
|
|
allgather,8,65536,ring,ll,4,2,16,-1,-1 # Small: ring (8B to 64K)
|
|
allgather,65537,16777216,ring,ll128,6,2,16,-1,-1 # Medium: ring (64K to 16M)
|
|
allgather,16777217,134217728,ring,simple,8,2,16,-1,-1 # Large: ring/simple (16M to 128M)
|
|
|
|
# 3 nodes, 24 ranks total
|
|
allgather,8,65536,ring,ll,4,2,24,-1,-1 # Small: ring (8B to 64K)
|
|
allgather,65537,16777216,ring,ll128,6,2,24,-1,-1 # Medium: ring (64K to 16M)
|
|
allgather,16777217,134217728,ring,simple,8,2,24,-1,-1 # Large: ring/simple (16M to 128M)
|
|
|
|
# 4 nodes, 32 ranks total
|
|
allgather,8,65536,ring,ll,4,2,32,-1,-1 # Small: ring (8B to 64K)
|
|
allgather,65537,16777216,ring,ll128,6,2,32,-1,-1 # Medium: ring (64K to 16M)
|
|
allgather,16777217,134217728,ring,simple,8,2,32,-1,-1 # Large: ring/simple (16M to 128M)
|
|
|
|
# 5 nodes, 40 ranks total
|
|
allgather,8,65536,ring,ll,4,2,40,-1,-1 # Small: ring (8B to 64K)
|
|
allgather,65537,16777216,ring,ll128,6,2,40,-1,-1 # Medium: ring (64K to 16M)
|
|
allgather,16777217,134217728,ring,simple,8,2,40,-1,-1 # Large: ring/simple (16M to 128M)
|
|
|
|
# Reduce configurations for multi-node setups - 8B-128M
|
|
# 2 nodes, 16 ranks total
|
|
reduce,8,65536,ring,ll,4,2,16,-1,-1 # Small: ring (8B to 64K)
|
|
reduce,65537,16777216,ring,ll128,6,2,16,-1,-1 # Medium: ring (64K to 16M)
|
|
reduce,16777217,134217728,ring,simple,8,2,16,-1,-1 # Large: ring/simple (16M to 128M)
|
|
|
|
# 3 nodes, 24 ranks total
|
|
reduce,8,65536,ring,ll,4,2,24,-1,-1 # Small: ring (8B to 64K)
|
|
reduce,65537,16777216,ring,ll128,6,2,24,-1,-1 # Medium: ring (64K to 16M)
|
|
reduce,16777217,134217728,ring,simple,8,2,24,-1,-1 # Large: ring/simple (16M to 128M)
|
|
|
|
# 4 nodes, 32 ranks total
|
|
reduce,8,65536,ring,ll,4,2,32,-1,-1 # Small: ring (8B to 64K)
|
|
reduce,65537,16777216,ring,ll128,6,2,32,-1,-1 # Medium: ring (64K to 16M)
|
|
reduce,16777217,134217728,ring,simple,8,2,32,-1,-1 # Large: ring/simple (16M to 128M)
|
|
|
|
# 5 nodes, 40 ranks total
|
|
reduce,8,65536,ring,ll,4,2,40,-1,-1 # Small: ring (8B to 64K)
|
|
reduce,65537,16777216,ring,ll128,6,2,40,-1,-1 # Medium: ring (64K to 16M)
|
|
reduce,16777217,134217728,ring,simple,8,2,40,-1,-1 # Large: ring/simple (16M to 128M)
|
|
|
|
# ReduceScatter configurations for multi-node setups - 8B-128M
|
|
# 2 nodes, 16 ranks total
|
|
reducescatter,8,65536,ring,ll,4,2,16,-1,-1 # Small: ring (8B to 64K)
|
|
reducescatter,65537,16777216,ring,ll128,6,2,16,-1,-1 # Medium: ring (64K to 16M)
|
|
reducescatter,16777217,134217728,ring,simple,8,2,16,-1,-1 # Large: ring/simple (16M to 128M)
|
|
|
|
# 3 nodes, 24 ranks total
|
|
reducescatter,8,65536,ring,ll,4,2,24,-1,-1 # Small: ring (8B to 64K)
|
|
reducescatter,65537,16777216,ring,ll128,6,2,24,-1,-1 # Medium: ring (64K to 16M)
|
|
reducescatter,16777217,134217728,ring,simple,8,2,24,-1,-1 # Large: ring/simple (16M to 128M)
|
|
|
|
# 4 nodes, 32 ranks total
|
|
reducescatter,8,65536,ring,ll,4,2,32,-1,-1 # Small: ring (8B to 64K)
|
|
reducescatter,65537,16777216,ring,ll128,6,2,32,-1,-1 # Medium: ring (64K to 16M)
|
|
reducescatter,16777217,134217728,ring,simple,8,2,32,-1,-1 # Large: ring/simple (16M to 128M)
|
|
|
|
# 5 nodes, 40 ranks total
|
|
reducescatter,8,65536,ring,ll,4,2,40,-1,-1 # Small: ring (8B to 64K)
|
|
reducescatter,65537,16777216,ring,ll128,6,2,40,-1,-1 # Medium: ring (64K to 16M)
|
|
reducescatter,16777217,134217728,ring,simple,8,2,40,-1,-1 # Large: ring/simple (16M to 128M)
|