From c2fc1d6809d8a250c0b693500e06d6c2c352c7d7 Mon Sep 17 00:00:00 2001
From: Pedram Alizadeh <pmohamma@amd.com>
Date: Thu, 21 Mar 2024 20:35:29 -0400
Subject: [PATCH] msccl algorithms tuning for alltoall on MI300 (#1120)

Co-authored-by: PedramAlizadeh <amd@pmohamma.com>
---
 ...all-8n-0-1kb.xml => alltoall-8n-0-9kb.xml} |   2 +-
 ...kb-1mb.xml => alltoall-8n-190kb-512kb.xml} |   2 +-
 .../msccl-algorithms/alltoall-8n-4kb-8kb.xml  | 740 ------------------
 ...-1mb-4mb.xml => alltoall-8n-512kb-7mb.xml} |   2 +-
 ...-4mb-16mb.xml => alltoall-8n-7mb-43mb.xml} |   2 +-
 ...-1kb-4kb.xml => alltoall-8n-9kb-190kb.xml} |   2 +-
 6 files changed, 5 insertions(+), 745 deletions(-)
 rename tools/msccl-algorithms/{alltoall-8n-0-1kb.xml => alltoall-8n-0-9kb.xml} (99%)
 rename tools/msccl-algorithms/{alltoall-8n-8kb-1mb.xml => alltoall-8n-190kb-512kb.xml} (99%)
 delete mode 100644 tools/msccl-algorithms/alltoall-8n-4kb-8kb.xml
 rename tools/msccl-algorithms/{alltoall-8n-1mb-4mb.xml => alltoall-8n-512kb-7mb.xml} (99%)
 rename tools/msccl-algorithms/{alltoall-8n-4mb-16mb.xml => alltoall-8n-7mb-43mb.xml} (99%)
 rename tools/msccl-algorithms/{alltoall-8n-1kb-4kb.xml => alltoall-8n-9kb-190kb.xml} (99%)

diff --git a/tools/msccl-algorithms/alltoall-8n-0-1kb.xml b/tools/msccl-algorithms/alltoall-8n-0-9kb.xml
similarity index 99%
rename from tools/msccl-algorithms/alltoall-8n-0-1kb.xml
rename to tools/msccl-algorithms/alltoall-8n-0-9kb.xml
index 7d3a57cb35..b2f388aaa6 100644
--- a/tools/msccl-algorithms/alltoall-8n-0-1kb.xml
+++ b/tools/msccl-algorithms/alltoall-8n-0-9kb.xml
@@ -1,6 +1,6 @@
 <!-- Copyright (c) Microsoft Corporation. -->
 <!-- Licensed under the MIT License. -->
-<algo name="alltoall_llm" proto="LL" nchannels="1" nchunksperloop="8" ngpus="8" coll="alltoall" inplace="0" outofplace="1" minBytes="0" maxBytes="1023">
+<algo name="alltoall_llm" proto="LL" nchannels="1" nchunksperloop="8" ngpus="8" coll="alltoall" inplace="0" outofplace="1" minBytes="0" maxBytes="9216">
   <gpu id="0" i_chunks="8" o_chunks="8" s_chunks="0">
     <tb id="0" send="-1" recv="-1" chan="0">
       <step s="0" type="cpy" srcbuf="i" srcoff="0" dstbuf="o" dstoff="0" cnt="1" depid="-1" deps="-1" hasdep="0"/>
diff --git a/tools/msccl-algorithms/alltoall-8n-8kb-1mb.xml b/tools/msccl-algorithms/alltoall-8n-190kb-512kb.xml
similarity index 99%
rename from tools/msccl-algorithms/alltoall-8n-8kb-1mb.xml
rename to tools/msccl-algorithms/alltoall-8n-190kb-512kb.xml
index 5817737540..cee7c6862e 100644
--- a/tools/msccl-algorithms/alltoall-8n-8kb-1mb.xml
+++ b/tools/msccl-algorithms/alltoall-8n-190kb-512kb.xml
@@ -1,6 +1,6 @@
 <!-- Copyright (c) Microsoft Corporation. -->
 <!-- Licensed under the MIT License. -->
-<algo name="alltoall_llm" proto="LL" nchannels="4" nchunksperloop="32" ngpus="8" coll="alltoall" inplace="0" outofplace="1" minBytes="8192" maxBytes="1048575">
+<algo name="alltoall_llm" proto="LL" nchannels="4" nchunksperloop="32" ngpus="8" coll="alltoall" inplace="0" outofplace="1" minBytes="194561" maxBytes="524288">
   <gpu id="0" i_chunks="32" o_chunks="32" s_chunks="0">
     <tb id="0" send="-1" recv="-1" chan="0">
       <step s="0" type="cpy" srcbuf="i" srcoff="0" dstbuf="o" dstoff="0" cnt="1" depid="-1" deps="-1" hasdep="0"/>
diff --git a/tools/msccl-algorithms/alltoall-8n-4kb-8kb.xml b/tools/msccl-algorithms/alltoall-8n-4kb-8kb.xml
deleted file mode 100644
index 3de528f979..0000000000
--- a/tools/msccl-algorithms/alltoall-8n-4kb-8kb.xml
+++ /dev/null
@@ -1,740 +0,0 @@
-<!-- Copyright (c) Microsoft Corporation. -->
-<!-- Licensed under the MIT License. -->
-<algo name="alltoall_llm" proto="LL" nchannels="16" nchunksperloop="16" ngpus="8" coll="alltoall" inplace="0" outofplace="1" minBytes="4096" maxBytes="8191">
-  <gpu id="0" i_chunks="16" o_chunks="16" s_chunks="0">
-    <tb id="0" send="-1" recv="-1" chan="0">
-      <step s="0" type="cpy" srcbuf="i" srcoff="0" dstbuf="o" dstoff="0" cnt="1" depid="-1" deps="-1" hasdep="0"/>
-    </tb>
-    <tb id="1" send="-1" recv="-1" chan="8">
-      <step s="0" type="cpy" srcbuf="i" srcoff="1" dstbuf="o" dstoff="1" cnt="1" depid="-1" deps="-1" hasdep="0"/>
-    </tb>
-    <tb id="2" send="-1" recv="1" chan="1">
-      <step s="0" type="r" srcbuf="i" srcoff="0" dstbuf="o" dstoff="2" cnt="1" depid="-1" deps="-1" hasdep="0"/>
-    </tb>
-    <tb id="3" send="-1" recv="1" chan="9">
-      <step s="0" type="r" srcbuf="i" srcoff="1" dstbuf="o" dstoff="3" cnt="1" depid="-1" deps="-1" hasdep="0"/>
-    </tb>
-    <tb id="4" send="-1" recv="2" chan="2">
-      <step s="0" type="r" srcbuf="i" srcoff="0" dstbuf="o" dstoff="4" cnt="1" depid="-1" deps="-1" hasdep="0"/>
-    </tb>
-    <tb id="5" send="-1" recv="2" chan="10">
-      <step s="0" type="r" srcbuf="i" srcoff="1" dstbuf="o" dstoff="5" cnt="1" depid="-1" deps="-1" hasdep="0"/>
-    </tb>
-    <tb id="6" send="-1" recv="3" chan="3">
-      <step s="0" type="r" srcbuf="i" srcoff="0" dstbuf="o" dstoff="6" cnt="1" depid="-1" deps="-1" hasdep="0"/>
-    </tb>
-    <tb id="7" send="-1" recv="3" chan="11">
-      <step s="0" type="r" srcbuf="i" srcoff="1" dstbuf="o" dstoff="7" cnt="1" depid="-1" deps="-1" hasdep="0"/>
-    </tb>
-    <tb id="8" send="-1" recv="4" chan="4">
-      <step s="0" type="r" srcbuf="i" srcoff="0" dstbuf="o" dstoff="8" cnt="1" depid="-1" deps="-1" hasdep="0"/>
-    </tb>
-    <tb id="9" send="-1" recv="4" chan="12">
-      <step s="0" type="r" srcbuf="i" srcoff="1" dstbuf="o" dstoff="9" cnt="1" depid="-1" deps="-1" hasdep="0"/>
-    </tb>
-    <tb id="10" send="-1" recv="5" chan="5">
-      <step s="0" type="r" srcbuf="i" srcoff="0" dstbuf="o" dstoff="10" cnt="1" depid="-1" deps="-1" hasdep="0"/>
-    </tb>
-    <tb id="11" send="-1" recv="5" chan="13">
-      <step s="0" type="r" srcbuf="i" srcoff="1" dstbuf="o" dstoff="11" cnt="1" depid="-1" deps="-1" hasdep="0"/>
-    </tb>
-    <tb id="12" send="-1" recv="6" chan="6">
-      <step s="0" type="r" srcbuf="i" srcoff="0" dstbuf="o" dstoff="12" cnt="1" depid="-1" deps="-1" hasdep="0"/>
-    </tb>
-    <tb id="13" send="-1" recv="6" chan="14">
-      <step s="0" type="r" srcbuf="i" srcoff="1" dstbuf="o" dstoff="13" cnt="1" depid="-1" deps="-1" hasdep="0"/>
-    </tb>
-    <tb id="14" send="-1" recv="7" chan="7">
-      <step s="0" type="r" srcbuf="i" srcoff="0" dstbuf="o" dstoff="14" cnt="1" depid="-1" deps="-1" hasdep="0"/>
-    </tb>
-    <tb id="15" send="-1" recv="7" chan="15">
-      <step s="0" type="r" srcbuf="i" srcoff="1" dstbuf="o" dstoff="15" cnt="1" depid="-1" deps="-1" hasdep="0"/>
-    </tb>
-    <tb id="16" send="1" recv="-1" chan="0">
-      <step s="0" type="s" srcbuf="i" srcoff="2" dstbuf="o" dstoff="0" cnt="1" depid="-1" deps="-1" hasdep="0"/>
-    </tb>
-    <tb id="17" send="1" recv="-1" chan="8">
-      <step s="0" type="s" srcbuf="i" srcoff="3" dstbuf="o" dstoff="1" cnt="1" depid="-1" deps="-1" hasdep="0"/>
-    </tb>
-    <tb id="18" send="2" recv="-1" chan="0">
-      <step s="0" type="s" srcbuf="i" srcoff="4" dstbuf="o" dstoff="0" cnt="1" depid="-1" deps="-1" hasdep="0"/>
-    </tb>
-    <tb id="19" send="2" recv="-1" chan="8">
-      <step s="0" type="s" srcbuf="i" srcoff="5" dstbuf="o" dstoff="1" cnt="1" depid="-1" deps="-1" hasdep="0"/>
-    </tb>
-    <tb id="20" send="3" recv="-1" chan="0">
-      <step s="0" type="s" srcbuf="i" srcoff="6" dstbuf="o" dstoff="0" cnt="1" depid="-1" deps="-1" hasdep="0"/>
-    </tb>
-    <tb id="21" send="3" recv="-1" chan="8">
-      <step s="0" type="s" srcbuf="i" srcoff="7" dstbuf="o" dstoff="1" cnt="1" depid="-1" deps="-1" hasdep="0"/>
-    </tb>
-    <tb id="22" send="4" recv="-1" chan="0">
-      <step s="0" type="s" srcbuf="i" srcoff="8" dstbuf="o" dstoff="0" cnt="1" depid="-1" deps="-1" hasdep="0"/>
-    </tb>
-    <tb id="23" send="4" recv="-1" chan="8">
-      <step s="0" type="s" srcbuf="i" srcoff="9" dstbuf="o" dstoff="1" cnt="1" depid="-1" deps="-1" hasdep="0"/>
-    </tb>
-    <tb id="24" send="5" recv="-1" chan="0">
-      <step s="0" type="s" srcbuf="i" srcoff="10" dstbuf="o" dstoff="0" cnt="1" depid="-1" deps="-1" hasdep="0"/>
-    </tb>
-    <tb id="25" send="5" recv="-1" chan="8">
-      <step s="0" type="s" srcbuf="i" srcoff="11" dstbuf="o" dstoff="1" cnt="1" depid="-1" deps="-1" hasdep="0"/>
-    </tb>
-    <tb id="26" send="6" recv="-1" chan="0">
-      <step s="0" type="s" srcbuf="i" srcoff="12" dstbuf="o" dstoff="0" cnt="1" depid="-1" deps="-1" hasdep="0"/>
-    </tb>
-    <tb id="27" send="6" recv="-1" chan="8">
-      <step s="0" type="s" srcbuf="i" srcoff="13" dstbuf="o" dstoff="1" cnt="1" depid="-1" deps="-1" hasdep="0"/>
-    </tb>
-    <tb id="28" send="7" recv="-1" chan="0">
-      <step s="0" type="s" srcbuf="i" srcoff="14" dstbuf="o" dstoff="0" cnt="1" depid="-1" deps="-1" hasdep="0"/>
-    </tb>
-    <tb id="29" send="7" recv="-1" chan="8">
-      <step s="0" type="s" srcbuf="i" srcoff="15" dstbuf="o" dstoff="1" cnt="1" depid="-1" deps="-1" hasdep="0"/>
-    </tb>
-  </gpu>
-  <gpu id="1" i_chunks="16" o_chunks="16" s_chunks="0">
-    <tb id="0" send="-1" recv="-1" chan="1">
-      <step s="0" type="cpy" srcbuf="i" srcoff="2" dstbuf="o" dstoff="2" cnt="1" depid="-1" deps="-1" hasdep="0"/>
-    </tb>
-    <tb id="1" send="-1" recv="-1" chan="9">
-      <step s="0" type="cpy" srcbuf="i" srcoff="3" dstbuf="o" dstoff="3" cnt="1" depid="-1" deps="-1" hasdep="0"/>
-    </tb>
-    <tb id="2" send="-1" recv="0" chan="0">
-      <step s="0" type="r" srcbuf="i" srcoff="2" dstbuf="o" dstoff="0" cnt="1" depid="-1" deps="-1" hasdep="0"/>
-    </tb>
-    <tb id="3" send="-1" recv="0" chan="8">
-      <step s="0" type="r" srcbuf="i" srcoff="3" dstbuf="o" dstoff="1" cnt="1" depid="-1" deps="-1" hasdep="0"/>
-    </tb>
-    <tb id="4" send="-1" recv="2" chan="2">
-      <step s="0" type="r" srcbuf="i" srcoff="2" dstbuf="o" dstoff="4" cnt="1" depid="-1" deps="-1" hasdep="0"/>
-    </tb>
-    <tb id="5" send="-1" recv="2" chan="10">
-      <step s="0" type="r" srcbuf="i" srcoff="3" dstbuf="o" dstoff="5" cnt="1" depid="-1" deps="-1" hasdep="0"/>
-    </tb>
-    <tb id="6" send="-1" recv="3" chan="3">
-      <step s="0" type="r" srcbuf="i" srcoff="2" dstbuf="o" dstoff="6" cnt="1" depid="-1" deps="-1" hasdep="0"/>
-    </tb>
-    <tb id="7" send="-1" recv="3" chan="11">
-      <step s="0" type="r" srcbuf="i" srcoff="3" dstbuf="o" dstoff="7" cnt="1" depid="-1" deps="-1" hasdep="0"/>
-    </tb>
-    <tb id="8" send="-1" recv="4" chan="4">
-      <step s="0" type="r" srcbuf="i" srcoff="2" dstbuf="o" dstoff="8" cnt="1" depid="-1" deps="-1" hasdep="0"/>
-    </tb>
-    <tb id="9" send="-1" recv="4" chan="12">
-      <step s="0" type="r" srcbuf="i" srcoff="3" dstbuf="o" dstoff="9" cnt="1" depid="-1" deps="-1" hasdep="0"/>
-    </tb>
-    <tb id="10" send="-1" recv="5" chan="5">
-      <step s="0" type="r" srcbuf="i" srcoff="2" dstbuf="o" dstoff="10" cnt="1" depid="-1" deps="-1" hasdep="0"/>
-    </tb>
-    <tb id="11" send="-1" recv="5" chan="13">
-      <step s="0" type="r" srcbuf="i" srcoff="3" dstbuf="o" dstoff="11" cnt="1" depid="-1" deps="-1" hasdep="0"/>
-    </tb>
-    <tb id="12" send="-1" recv="6" chan="6">
-      <step s="0" type="r" srcbuf="i" srcoff="2" dstbuf="o" dstoff="12" cnt="1" depid="-1" deps="-1" hasdep="0"/>
-    </tb>
-    <tb id="13" send="-1" recv="6" chan="14">
-      <step s="0" type="r" srcbuf="i" srcoff="3" dstbuf="o" dstoff="13" cnt="1" depid="-1" deps="-1" hasdep="0"/>
-    </tb>
-    <tb id="14" send="-1" recv="7" chan="7">
-      <step s="0" type="r" srcbuf="i" srcoff="2" dstbuf="o" dstoff="14" cnt="1" depid="-1" deps="-1" hasdep="0"/>
-    </tb>
-    <tb id="15" send="-1" recv="7" chan="15">
-      <step s="0" type="r" srcbuf="i" srcoff="3" dstbuf="o" dstoff="15" cnt="1" depid="-1" deps="-1" hasdep="0"/>
-    </tb>
-    <tb id="16" send="0" recv="-1" chan="1">
-      <step s="0" type="s" srcbuf="i" srcoff="0" dstbuf="o" dstoff="2" cnt="1" depid="-1" deps="-1" hasdep="0"/>
-    </tb>
-    <tb id="17" send="0" recv="-1" chan="9">
-      <step s="0" type="s" srcbuf="i" srcoff="1" dstbuf="o" dstoff="3" cnt="1" depid="-1" deps="-1" hasdep="0"/>
-    </tb>
-    <tb id="18" send="2" recv="-1" chan="1">
-      <step s="0" type="s" srcbuf="i" srcoff="4" dstbuf="o" dstoff="2" cnt="1" depid="-1" deps="-1" hasdep="0"/>
-    </tb>
-    <tb id="19" send="2" recv="-1" chan="9">
-      <step s="0" type="s" srcbuf="i" srcoff="5" dstbuf="o" dstoff="3" cnt="1" depid="-1" deps="-1" hasdep="0"/>
-    </tb>
-    <tb id="20" send="3" recv="-1" chan="1">
-      <step s="0" type="s" srcbuf="i" srcoff="6" dstbuf="o" dstoff="2" cnt="1" depid="-1" deps="-1" hasdep="0"/>
-    </tb>
-    <tb id="21" send="3" recv="-1" chan="9">
-      <step s="0" type="s" srcbuf="i" srcoff="7" dstbuf="o" dstoff="3" cnt="1" depid="-1" deps="-1" hasdep="0"/>
-    </tb>
-    <tb id="22" send="4" recv="-1" chan="1">
-      <step s="0" type="s" srcbuf="i" srcoff="8" dstbuf="o" dstoff="2" cnt="1" depid="-1" deps="-1" hasdep="0"/>
-    </tb>
-    <tb id="23" send="4" recv="-1" chan="9">
-      <step s="0" type="s" srcbuf="i" srcoff="9" dstbuf="o" dstoff="3" cnt="1" depid="-1" deps="-1" hasdep="0"/>
-    </tb>
-    <tb id="24" send="5" recv="-1" chan="1">
-      <step s="0" type="s" srcbuf="i" srcoff="10" dstbuf="o" dstoff="2" cnt="1" depid="-1" deps="-1" hasdep="0"/>
-    </tb>
-    <tb id="25" send="5" recv="-1" chan="9">
-      <step s="0" type="s" srcbuf="i" srcoff="11" dstbuf="o" dstoff="3" cnt="1" depid="-1" deps="-1" hasdep="0"/>
-    </tb>
-    <tb id="26" send="6" recv="-1" chan="1">
-      <step s="0" type="s" srcbuf="i" srcoff="12" dstbuf="o" dstoff="2" cnt="1" depid="-1" deps="-1" hasdep="0"/>
-    </tb>
-    <tb id="27" send="6" recv="-1" chan="9">
-      <step s="0" type="s" srcbuf="i" srcoff="13" dstbuf="o" dstoff="3" cnt="1" depid="-1" deps="-1" hasdep="0"/>
-    </tb>
-    <tb id="28" send="7" recv="-1" chan="1">
-      <step s="0" type="s" srcbuf="i" srcoff="14" dstbuf="o" dstoff="2" cnt="1" depid="-1" deps="-1" hasdep="0"/>
-    </tb>
-    <tb id="29" send="7" recv="-1" chan="9">
-      <step s="0" type="s" srcbuf="i" srcoff="15" dstbuf="o" dstoff="3" cnt="1" depid="-1" deps="-1" hasdep="0"/>
-    </tb>
-  </gpu>
-  <gpu id="2" i_chunks="16" o_chunks="16" s_chunks="0">
-    <tb id="0" send="-1" recv="-1" chan="2">
-      <step s="0" type="cpy" srcbuf="i" srcoff="4" dstbuf="o" dstoff="4" cnt="1" depid="-1" deps="-1" hasdep="0"/>
-    </tb>
-    <tb id="1" send="-1" recv="-1" chan="10">
-      <step s="0" type="cpy" srcbuf="i" srcoff="5" dstbuf="o" dstoff="5" cnt="1" depid="-1" deps="-1" hasdep="0"/>
-    </tb>
-    <tb id="2" send="-1" recv="0" chan="0">
-      <step s="0" type="r" srcbuf="i" srcoff="4" dstbuf="o" dstoff="0" cnt="1" depid="-1" deps="-1" hasdep="0"/>
-    </tb>
-    <tb id="3" send="-1" recv="0" chan="8">
-      <step s="0" type="r" srcbuf="i" srcoff="5" dstbuf="o" dstoff="1" cnt="1" depid="-1" deps="-1" hasdep="0"/>
-    </tb>
-    <tb id="4" send="-1" recv="1" chan="1">
-      <step s="0" type="r" srcbuf="i" srcoff="4" dstbuf="o" dstoff="2" cnt="1" depid="-1" deps="-1" hasdep="0"/>
-    </tb>
-    <tb id="5" send="-1" recv="1" chan="9">
-      <step s="0" type="r" srcbuf="i" srcoff="5" dstbuf="o" dstoff="3" cnt="1" depid="-1" deps="-1" hasdep="0"/>
-    </tb>
-    <tb id="6" send="-1" recv="3" chan="3">
-      <step s="0" type="r" srcbuf="i" srcoff="4" dstbuf="o" dstoff="6" cnt="1" depid="-1" deps="-1" hasdep="0"/>
-    </tb>
-    <tb id="7" send="-1" recv="3" chan="11">
-      <step s="0" type="r" srcbuf="i" srcoff="5" dstbuf="o" dstoff="7" cnt="1" depid="-1" deps="-1" hasdep="0"/>
-    </tb>
-    <tb id="8" send="-1" recv="4" chan="4">
-      <step s="0" type="r" srcbuf="i" srcoff="4" dstbuf="o" dstoff="8" cnt="1" depid="-1" deps="-1" hasdep="0"/>
-    </tb>
-    <tb id="9" send="-1" recv="4" chan="12">
-      <step s="0" type="r" srcbuf="i" srcoff="5" dstbuf="o" dstoff="9" cnt="1" depid="-1" deps="-1" hasdep="0"/>
-    </tb>
-    <tb id="10" send="-1" recv="5" chan="5">
-      <step s="0" type="r" srcbuf="i" srcoff="4" dstbuf="o" dstoff="10" cnt="1" depid="-1" deps="-1" hasdep="0"/>
-    </tb>
-    <tb id="11" send="-1" recv="5" chan="13">
-      <step s="0" type="r" srcbuf="i" srcoff="5" dstbuf="o" dstoff="11" cnt="1" depid="-1" deps="-1" hasdep="0"/>
-    </tb>
-    <tb id="12" send="-1" recv="6" chan="6">
-      <step s="0" type="r" srcbuf="i" srcoff="4" dstbuf="o" dstoff="12" cnt="1" depid="-1" deps="-1" hasdep="0"/>
-    </tb>
-    <tb id="13" send="-1" recv="6" chan="14">
-      <step s="0" type="r" srcbuf="i" srcoff="5" dstbuf="o" dstoff="13" cnt="1" depid="-1" deps="-1" hasdep="0"/>
-    </tb>
-    <tb id="14" send="-1" recv="7" chan="7">
-      <step s="0" type="r" srcbuf="i" srcoff="4" dstbuf="o" dstoff="14" cnt="1" depid="-1" deps="-1" hasdep="0"/>
-    </tb>
-    <tb id="15" send="-1" recv="7" chan="15">
-      <step s="0" type="r" srcbuf="i" srcoff="5" dstbuf="o" dstoff="15" cnt="1" depid="-1" deps="-1" hasdep="0"/>
-    </tb>
-    <tb id="16" send="0" recv="-1" chan="2">
-      <step s="0" type="s" srcbuf="i" srcoff="0" dstbuf="o" dstoff="4" cnt="1" depid="-1" deps="-1" hasdep="0"/>
-    </tb>
-    <tb id="17" send="0" recv="-1" chan="10">
-      <step s="0" type="s" srcbuf="i" srcoff="1" dstbuf="o" dstoff="5" cnt="1" depid="-1" deps="-1" hasdep="0"/>
-    </tb>
-    <tb id="18" send="1" recv="-1" chan="2">
-      <step s="0" type="s" srcbuf="i" srcoff="2" dstbuf="o" dstoff="4" cnt="1" depid="-1" deps="-1" hasdep="0"/>
-    </tb>
-    <tb id="19" send="1" recv="-1" chan="10">
-      <step s="0" type="s" srcbuf="i" srcoff="3" dstbuf="o" dstoff="5" cnt="1" depid="-1" deps="-1" hasdep="0"/>
-    </tb>
-    <tb id="20" send="3" recv="-1" chan="2">
-      <step s="0" type="s" srcbuf="i" srcoff="6" dstbuf="o" dstoff="4" cnt="1" depid="-1" deps="-1" hasdep="0"/>
-    </tb>
-    <tb id="21" send="3" recv="-1" chan="10">
-      <step s="0" type="s" srcbuf="i" srcoff="7" dstbuf="o" dstoff="5" cnt="1" depid="-1" deps="-1" hasdep="0"/>
-    </tb>
-    <tb id="22" send="4" recv="-1" chan="2">
-      <step s="0" type="s" srcbuf="i" srcoff="8" dstbuf="o" dstoff="4" cnt="1" depid="-1" deps="-1" hasdep="0"/>
-    </tb>
-    <tb id="23" send="4" recv="-1" chan="10">
-      <step s="0" type="s" srcbuf="i" srcoff="9" dstbuf="o" dstoff="5" cnt="1" depid="-1" deps="-1" hasdep="0"/>
-    </tb>
-    <tb id="24" send="5" recv="-1" chan="2">
-      <step s="0" type="s" srcbuf="i" srcoff="10" dstbuf="o" dstoff="4" cnt="1" depid="-1" deps="-1" hasdep="0"/>
-    </tb>
-    <tb id="25" send="5" recv="-1" chan="10">
-      <step s="0" type="s" srcbuf="i" srcoff="11" dstbuf="o" dstoff="5" cnt="1" depid="-1" deps="-1" hasdep="0"/>
-    </tb>
-    <tb id="26" send="6" recv="-1" chan="2">
-      <step s="0" type="s" srcbuf="i" srcoff="12" dstbuf="o" dstoff="4" cnt="1" depid="-1" deps="-1" hasdep="0"/>
-    </tb>
-    <tb id="27" send="6" recv="-1" chan="10">
-      <step s="0" type="s" srcbuf="i" srcoff="13" dstbuf="o" dstoff="5" cnt="1" depid="-1" deps="-1" hasdep="0"/>
-    </tb>
-    <tb id="28" send="7" recv="-1" chan="2">
-      <step s="0" type="s" srcbuf="i" srcoff="14" dstbuf="o" dstoff="4" cnt="1" depid="-1" deps="-1" hasdep="0"/>
-    </tb>
-    <tb id="29" send="7" recv="-1" chan="10">
-      <step s="0" type="s" srcbuf="i" srcoff="15" dstbuf="o" dstoff="5" cnt="1" depid="-1" deps="-1" hasdep="0"/>
-    </tb>
-  </gpu>
-  <gpu id="3" i_chunks="16" o_chunks="16" s_chunks="0">
-    <tb id="0" send="-1" recv="-1" chan="3">
-      <step s="0" type="cpy" srcbuf="i" srcoff="6" dstbuf="o" dstoff="6" cnt="1" depid="-1" deps="-1" hasdep="0"/>
-    </tb>
-    <tb id="1" send="-1" recv="-1" chan="11">
-      <step s="0" type="cpy" srcbuf="i" srcoff="7" dstbuf="o" dstoff="7" cnt="1" depid="-1" deps="-1" hasdep="0"/>
-    </tb>
-    <tb id="2" send="-1" recv="0" chan="0">
-      <step s="0" type="r" srcbuf="i" srcoff="6" dstbuf="o" dstoff="0" cnt="1" depid="-1" deps="-1" hasdep="0"/>
-    </tb>
-    <tb id="3" send="-1" recv="0" chan="8">
-      <step s="0" type="r" srcbuf="i" srcoff="7" dstbuf="o" dstoff="1" cnt="1" depid="-1" deps="-1" hasdep="0"/>
-    </tb>
-    <tb id="4" send="-1" recv="1" chan="1">
-      <step s="0" type="r" srcbuf="i" srcoff="6" dstbuf="o" dstoff="2" cnt="1" depid="-1" deps="-1" hasdep="0"/>
-    </tb>
-    <tb id="5" send="-1" recv="1" chan="9">
-      <step s="0" type="r" srcbuf="i" srcoff="7" dstbuf="o" dstoff="3" cnt="1" depid="-1" deps="-1" hasdep="0"/>
-    </tb>
-    <tb id="6" send="-1" recv="2" chan="2">
-      <step s="0" type="r" srcbuf="i" srcoff="6" dstbuf="o" dstoff="4" cnt="1" depid="-1" deps="-1" hasdep="0"/>
-    </tb>
-    <tb id="7" send="-1" recv="2" chan="10">
-      <step s="0" type="r" srcbuf="i" srcoff="7" dstbuf="o" dstoff="5" cnt="1" depid="-1" deps="-1" hasdep="0"/>
-    </tb>
-    <tb id="8" send="-1" recv="4" chan="4">
-      <step s="0" type="r" srcbuf="i" srcoff="6" dstbuf="o" dstoff="8" cnt="1" depid="-1" deps="-1" hasdep="0"/>
-    </tb>
-    <tb id="9" send="-1" recv="4" chan="12">
-      <step s="0" type="r" srcbuf="i" srcoff="7" dstbuf="o" dstoff="9" cnt="1" depid="-1" deps="-1" hasdep="0"/>
-    </tb>
-    <tb id="10" send="-1" recv="5" chan="5">
-      <step s="0" type="r" srcbuf="i" srcoff="6" dstbuf="o" dstoff="10" cnt="1" depid="-1" deps="-1" hasdep="0"/>
-    </tb>
-    <tb id="11" send="-1" recv="5" chan="13">
-      <step s="0" type="r" srcbuf="i" srcoff="7" dstbuf="o" dstoff="11" cnt="1" depid="-1" deps="-1" hasdep="0"/>
-    </tb>
-    <tb id="12" send="-1" recv="6" chan="6">
-      <step s="0" type="r" srcbuf="i" srcoff="6" dstbuf="o" dstoff="12" cnt="1" depid="-1" deps="-1" hasdep="0"/>
-    </tb>
-    <tb id="13" send="-1" recv="6" chan="14">
-      <step s="0" type="r" srcbuf="i" srcoff="7" dstbuf="o" dstoff="13" cnt="1" depid="-1" deps="-1" hasdep="0"/>
-    </tb>
-    <tb id="14" send="-1" recv="7" chan="7">
-      <step s="0" type="r" srcbuf="i" srcoff="6" dstbuf="o" dstoff="14" cnt="1" depid="-1" deps="-1" hasdep="0"/>
-    </tb>
-    <tb id="15" send="-1" recv="7" chan="15">
-      <step s="0" type="r" srcbuf="i" srcoff="7" dstbuf="o" dstoff="15" cnt="1" depid="-1" deps="-1" hasdep="0"/>
-    </tb>
-    <tb id="16" send="0" recv="-1" chan="3">
-      <step s="0" type="s" srcbuf="i" srcoff="0" dstbuf="o" dstoff="6" cnt="1" depid="-1" deps="-1" hasdep="0"/>
-    </tb>
-    <tb id="17" send="0" recv="-1" chan="11">
-      <step s="0" type="s" srcbuf="i" srcoff="1" dstbuf="o" dstoff="7" cnt="1" depid="-1" deps="-1" hasdep="0"/>
-    </tb>
-    <tb id="18" send="1" recv="-1" chan="3">
-      <step s="0" type="s" srcbuf="i" srcoff="2" dstbuf="o" dstoff="6" cnt="1" depid="-1" deps="-1" hasdep="0"/>
-    </tb>
-    <tb id="19" send="1" recv="-1" chan="11">
-      <step s="0" type="s" srcbuf="i" srcoff="3" dstbuf="o" dstoff="7" cnt="1" depid="-1" deps="-1" hasdep="0"/>
-    </tb>
-    <tb id="20" send="2" recv="-1" chan="3">
-      <step s="0" type="s" srcbuf="i" srcoff="4" dstbuf="o" dstoff="6" cnt="1" depid="-1" deps="-1" hasdep="0"/>
-    </tb>
-    <tb id="21" send="2" recv="-1" chan="11">
-      <step s="0" type="s" srcbuf="i" srcoff="5" dstbuf="o" dstoff="7" cnt="1" depid="-1" deps="-1" hasdep="0"/>
-    </tb>
-    <tb id="22" send="4" recv="-1" chan="3">
-      <step s="0" type="s" srcbuf="i" srcoff="8" dstbuf="o" dstoff="6" cnt="1" depid="-1" deps="-1" hasdep="0"/>
-    </tb>
-    <tb id="23" send="4" recv="-1" chan="11">
-      <step s="0" type="s" srcbuf="i" srcoff="9" dstbuf="o" dstoff="7" cnt="1" depid="-1" deps="-1" hasdep="0"/>
-    </tb>
-    <tb id="24" send="5" recv="-1" chan="3">
-      <step s="0" type="s" srcbuf="i" srcoff="10" dstbuf="o" dstoff="6" cnt="1" depid="-1" deps="-1" hasdep="0"/>
-    </tb>
-    <tb id="25" send="5" recv="-1" chan="11">
-      <step s="0" type="s" srcbuf="i" srcoff="11" dstbuf="o" dstoff="7" cnt="1" depid="-1" deps="-1" hasdep="0"/>
-    </tb>
-    <tb id="26" send="6" recv="-1" chan="3">
-      <step s="0" type="s" srcbuf="i" srcoff="12" dstbuf="o" dstoff="6" cnt="1" depid="-1" deps="-1" hasdep="0"/>
-    </tb>
-    <tb id="27" send="6" recv="-1" chan="11">
-      <step s="0" type="s" srcbuf="i" srcoff="13" dstbuf="o" dstoff="7" cnt="1" depid="-1" deps="-1" hasdep="0"/>
-    </tb>
-    <tb id="28" send="7" recv="-1" chan="3">
-      <step s="0" type="s" srcbuf="i" srcoff="14" dstbuf="o" dstoff="6" cnt="1" depid="-1" deps="-1" hasdep="0"/>
-    </tb>
-    <tb id="29" send="7" recv="-1" chan="11">
-      <step s="0" type="s" srcbuf="i" srcoff="15" dstbuf="o" dstoff="7" cnt="1" depid="-1" deps="-1" hasdep="0"/>
-    </tb>
-  </gpu>
-  <gpu id="4" i_chunks="16" o_chunks="16" s_chunks="0">
-    <tb id="0" send="-1" recv="-1" chan="4">
-      <step s="0" type="cpy" srcbuf="i" srcoff="8" dstbuf="o" dstoff="8" cnt="1" depid="-1" deps="-1" hasdep="0"/>
-    </tb>
-    <tb id="1" send="-1" recv="-1" chan="12">
-      <step s="0" type="cpy" srcbuf="i" srcoff="9" dstbuf="o" dstoff="9" cnt="1" depid="-1" deps="-1" hasdep="0"/>
-    </tb>
-    <tb id="2" send="-1" recv="0" chan="0">
-      <step s="0" type="r" srcbuf="i" srcoff="8" dstbuf="o" dstoff="0" cnt="1" depid="-1" deps="-1" hasdep="0"/>
-    </tb>
-    <tb id="3" send="-1" recv="0" chan="8">
-      <step s="0" type="r" srcbuf="i" srcoff="9" dstbuf="o" dstoff="1" cnt="1" depid="-1" deps="-1" hasdep="0"/>
-    </tb>
-    <tb id="4" send="-1" recv="1" chan="1">
-      <step s="0" type="r" srcbuf="i" srcoff="8" dstbuf="o" dstoff="2" cnt="1" depid="-1" deps="-1" hasdep="0"/>
-    </tb>
-    <tb id="5" send="-1" recv="1" chan="9">
-      <step s="0" type="r" srcbuf="i" srcoff="9" dstbuf="o" dstoff="3" cnt="1" depid="-1" deps="-1" hasdep="0"/>
-    </tb>
-    <tb id="6" send="-1" recv="2" chan="2">
-      <step s="0" type="r" srcbuf="i" srcoff="8" dstbuf="o" dstoff="4" cnt="1" depid="-1" deps="-1" hasdep="0"/>
-    </tb>
-    <tb id="7" send="-1" recv="2" chan="10">
-      <step s="0" type="r" srcbuf="i" srcoff="9" dstbuf="o" dstoff="5" cnt="1" depid="-1" deps="-1" hasdep="0"/>
-    </tb>
-    <tb id="8" send="-1" recv="3" chan="3">
-      <step s="0" type="r" srcbuf="i" srcoff="8" dstbuf="o" dstoff="6" cnt="1" depid="-1" deps="-1" hasdep="0"/>
-    </tb>
-    <tb id="9" send="-1" recv="3" chan="11">
-      <step s="0" type="r" srcbuf="i" srcoff="9" dstbuf="o" dstoff="7" cnt="1" depid="-1" deps="-1" hasdep="0"/>
-    </tb>
-    <tb id="10" send="-1" recv="5" chan="5">
-      <step s="0" type="r" srcbuf="i" srcoff="8" dstbuf="o" dstoff="10" cnt="1" depid="-1" deps="-1" hasdep="0"/>
-    </tb>
-    <tb id="11" send="-1" recv="5" chan="13">
-      <step s="0" type="r" srcbuf="i" srcoff="9" dstbuf="o" dstoff="11" cnt="1" depid="-1" deps="-1" hasdep="0"/>
-    </tb>
-    <tb id="12" send="-1" recv="6" chan="6">
-      <step s="0" type="r" srcbuf="i" srcoff="8" dstbuf="o" dstoff="12" cnt="1" depid="-1" deps="-1" hasdep="0"/>
-    </tb>
-    <tb id="13" send="-1" recv="6" chan="14">
-      <step s="0" type="r" srcbuf="i" srcoff="9" dstbuf="o" dstoff="13" cnt="1" depid="-1" deps="-1" hasdep="0"/>
-    </tb>
-    <tb id="14" send="-1" recv="7" chan="7">
-      <step s="0" type="r" srcbuf="i" srcoff="8" dstbuf="o" dstoff="14" cnt="1" depid="-1" deps="-1" hasdep="0"/>
-    </tb>
-    <tb id="15" send="-1" recv="7" chan="15">
-      <step s="0" type="r" srcbuf="i" srcoff="9" dstbuf="o" dstoff="15" cnt="1" depid="-1" deps="-1" hasdep="0"/>
-    </tb>
-    <tb id="16" send="0" recv="-1" chan="4">
-      <step s="0" type="s" srcbuf="i" srcoff="0" dstbuf="o" dstoff="8" cnt="1" depid="-1" deps="-1" hasdep="0"/>
-    </tb>
-    <tb id="17" send="0" recv="-1" chan="12">
-      <step s="0" type="s" srcbuf="i" srcoff="1" dstbuf="o" dstoff="9" cnt="1" depid="-1" deps="-1" hasdep="0"/>
-    </tb>
-    <tb id="18" send="1" recv="-1" chan="4">
-      <step s="0" type="s" srcbuf="i" srcoff="2" dstbuf="o" dstoff="8" cnt="1" depid="-1" deps="-1" hasdep="0"/>
-    </tb>
-    <tb id="19" send="1" recv="-1" chan="12">
-      <step s="0" type="s" srcbuf="i" srcoff="3" dstbuf="o" dstoff="9" cnt="1" depid="-1" deps="-1" hasdep="0"/>
-    </tb>
-    <tb id="20" send="2" recv="-1" chan="4">
-      <step s="0" type="s" srcbuf="i" srcoff="4" dstbuf="o" dstoff="8" cnt="1" depid="-1" deps="-1" hasdep="0"/>
-    </tb>
-    <tb id="21" send="2" recv="-1" chan="12">
-      <step s="0" type="s" srcbuf="i" srcoff="5" dstbuf="o" dstoff="9" cnt="1" depid="-1" deps="-1" hasdep="0"/>
-    </tb>
-    <tb id="22" send="3" recv="-1" chan="4">
-      <step s="0" type="s" srcbuf="i" srcoff="6" dstbuf="o" dstoff="8" cnt="1" depid="-1" deps="-1" hasdep="0"/>
-    </tb>
-    <tb id="23" send="3" recv="-1" chan="12">
-      <step s="0" type="s" srcbuf="i" srcoff="7" dstbuf="o" dstoff="9" cnt="1" depid="-1" deps="-1" hasdep="0"/>
-    </tb>
-    <tb id="24" send="5" recv="-1" chan="4">
-      <step s="0" type="s" srcbuf="i" srcoff="10" dstbuf="o" dstoff="8" cnt="1" depid="-1" deps="-1" hasdep="0"/>
-    </tb>
-    <tb id="25" send="5" recv="-1" chan="12">
-      <step s="0" type="s" srcbuf="i" srcoff="11" dstbuf="o" dstoff="9" cnt="1" depid="-1" deps="-1" hasdep="0"/>
-    </tb>
-    <tb id="26" send="6" recv="-1" chan="4">
-      <step s="0" type="s" srcbuf="i" srcoff="12" dstbuf="o" dstoff="8" cnt="1" depid="-1" deps="-1" hasdep="0"/>
-    </tb>
-    <tb id="27" send="6" recv="-1" chan="12">
-      <step s="0" type="s" srcbuf="i" srcoff="13" dstbuf="o" dstoff="9" cnt="1" depid="-1" deps="-1" hasdep="0"/>
-    </tb>
-    <tb id="28" send="7" recv="-1" chan="4">
-      <step s="0" type="s" srcbuf="i" srcoff="14" dstbuf="o" dstoff="8" cnt="1" depid="-1" deps="-1" hasdep="0"/>
-    </tb>
-    <tb id="29" send="7" recv="-1" chan="12">
-      <step s="0" type="s" srcbuf="i" srcoff="15" dstbuf="o" dstoff="9" cnt="1" depid="-1" deps="-1" hasdep="0"/>
-    </tb>
-  </gpu>
-  <gpu id="5" i_chunks="16" o_chunks="16" s_chunks="0">
-    <tb id="0" send="-1" recv="-1" chan="5">
-      <step s="0" type="cpy" srcbuf="i" srcoff="10" dstbuf="o" dstoff="10" cnt="1" depid="-1" deps="-1" hasdep="0"/>
-    </tb>
-    <tb id="1" send="-1" recv="-1" chan="13">
-      <step s="0" type="cpy" srcbuf="i" srcoff="11" dstbuf="o" dstoff="11" cnt="1" depid="-1" deps="-1" hasdep="0"/>
-    </tb>
-    <tb id="2" send="-1" recv="0" chan="0">
-      <step s="0" type="r" srcbuf="i" srcoff="10" dstbuf="o" dstoff="0" cnt="1" depid="-1" deps="-1" hasdep="0"/>
-    </tb>
-    <tb id="3" send="-1" recv="0" chan="8">
-      <step s="0" type="r" srcbuf="i" srcoff="11" dstbuf="o" dstoff="1" cnt="1" depid="-1" deps="-1" hasdep="0"/>
-    </tb>
-    <tb id="4" send="-1" recv="1" chan="1">
-      <step s="0" type="r" srcbuf="i" srcoff="10" dstbuf="o" dstoff="2" cnt="1" depid="-1" deps="-1" hasdep="0"/>
-    </tb>
-    <tb id="5" send="-1" recv="1" chan="9">
-      <step s="0" type="r" srcbuf="i" srcoff="11" dstbuf="o" dstoff="3" cnt="1" depid="-1" deps="-1" hasdep="0"/>
-    </tb>
-    <tb id="6" send="-1" recv="2" chan="2">
-      <step s="0" type="r" srcbuf="i" srcoff="10" dstbuf="o" dstoff="4" cnt="1" depid="-1" deps="-1" hasdep="0"/>
-    </tb>
-    <tb id="7" send="-1" recv="2" chan="10">
-      <step s="0" type="r" srcbuf="i" srcoff="11" dstbuf="o" dstoff="5" cnt="1" depid="-1" deps="-1" hasdep="0"/>
-    </tb>
-    <tb id="8" send="-1" recv="3" chan="3">
-      <step s="0" type="r" srcbuf="i" srcoff="10" dstbuf="o" dstoff="6" cnt="1" depid="-1" deps="-1" hasdep="0"/>
-    </tb>
-    <tb id="9" send="-1" recv="3" chan="11">
-      <step s="0" type="r" srcbuf="i" srcoff="11" dstbuf="o" dstoff="7" cnt="1" depid="-1" deps="-1" hasdep="0"/>
-    </tb>
-    <tb id="10" send="-1" recv="4" chan="4">
-      <step s="0" type="r" srcbuf="i" srcoff="10" dstbuf="o" dstoff="8" cnt="1" depid="-1" deps="-1" hasdep="0"/>
-    </tb>
-    <tb id="11" send="-1" recv="4" chan="12">
-      <step s="0" type="r" srcbuf="i" srcoff="11" dstbuf="o" dstoff="9" cnt="1" depid="-1" deps="-1" hasdep="0"/>
-    </tb>
-    <tb id="12" send="-1" recv="6" chan="6">
-      <step s="0" type="r" srcbuf="i" srcoff="10" dstbuf="o" dstoff="12" cnt="1" depid="-1" deps="-1" hasdep="0"/>
-    </tb>
-    <tb id="13" send="-1" recv="6" chan="14">
-      <step s="0" type="r" srcbuf="i" srcoff="11" dstbuf="o" dstoff="13" cnt="1" depid="-1" deps="-1" hasdep="0"/>
-    </tb>
-    <tb id="14" send="-1" recv="7" chan="7">
-      <step s="0" type="r" srcbuf="i" srcoff="10" dstbuf="o" dstoff="14" cnt="1" depid="-1" deps="-1" hasdep="0"/>
-    </tb>
-    <tb id="15" send="-1" recv="7" chan="15">
-      <step s="0" type="r" srcbuf="i" srcoff="11" dstbuf="o" dstoff="15" cnt="1" depid="-1" deps="-1" hasdep="0"/>
-    </tb>
-    <tb id="16" send="0" recv="-1" chan="5">
-      <step s="0" type="s" srcbuf="i" srcoff="0" dstbuf="o" dstoff="10" cnt="1" depid="-1" deps="-1" hasdep="0"/>
-    </tb>
-    <tb id="17" send="0" recv="-1" chan="13">
-      <step s="0" type="s" srcbuf="i" srcoff="1" dstbuf="o" dstoff="11" cnt="1" depid="-1" deps="-1" hasdep="0"/>
-    </tb>
-    <tb id="18" send="1" recv="-1" chan="5">
-      <step s="0" type="s" srcbuf="i" srcoff="2" dstbuf="o" dstoff="10" cnt="1" depid="-1" deps="-1" hasdep="0"/>
-    </tb>
-    <tb id="19" send="1" recv="-1" chan="13">
-      <step s="0" type="s" srcbuf="i" srcoff="3" dstbuf="o" dstoff="11" cnt="1" depid="-1" deps="-1" hasdep="0"/>
-    </tb>
-    <tb id="20" send="2" recv="-1" chan="5">
-      <step s="0" type="s" srcbuf="i" srcoff="4" dstbuf="o" dstoff="10" cnt="1" depid="-1" deps="-1" hasdep="0"/>
-    </tb>
-    <tb id="21" send="2" recv="-1" chan="13">
-      <step s="0" type="s" srcbuf="i" srcoff="5" dstbuf="o" dstoff="11" cnt="1" depid="-1" deps="-1" hasdep="0"/>
-    </tb>
-    <tb id="22" send="3" recv="-1" chan="5">
-      <step s="0" type="s" srcbuf="i" srcoff="6" dstbuf="o" dstoff="10" cnt="1" depid="-1" deps="-1" hasdep="0"/>
-    </tb>
-    <tb id="23" send="3" recv="-1" chan="13">
-      <step s="0" type="s" srcbuf="i" srcoff="7" dstbuf="o" dstoff="11" cnt="1" depid="-1" deps="-1" hasdep="0"/>
-    </tb>
-    <tb id="24" send="4" recv="-1" chan="5">
-      <step s="0" type="s" srcbuf="i" srcoff="8" dstbuf="o" dstoff="10" cnt="1" depid="-1" deps="-1" hasdep="0"/>
-    </tb>
-    <tb id="25" send="4" recv="-1" chan="13">
-      <step s="0" type="s" srcbuf="i" srcoff="9" dstbuf="o" dstoff="11" cnt="1" depid="-1" deps="-1" hasdep="0"/>
-    </tb>
-    <tb id="26" send="6" recv="-1" chan="5">
-      <step s="0" type="s" srcbuf="i" srcoff="12" dstbuf="o" dstoff="10" cnt="1" depid="-1" deps="-1" hasdep="0"/>
-    </tb>
-    <tb id="27" send="6" recv="-1" chan="13">
-      <step s="0" type="s" srcbuf="i" srcoff="13" dstbuf="o" dstoff="11" cnt="1" depid="-1" deps="-1" hasdep="0"/>
-    </tb>
-    <tb id="28" send="7" recv="-1" chan="5">
-      <step s="0" type="s" srcbuf="i" srcoff="14" dstbuf="o" dstoff="10" cnt="1" depid="-1" deps="-1" hasdep="0"/>
-    </tb>
-    <tb id="29" send="7" recv="-1" chan="13">
-      <step s="0" type="s" srcbuf="i" srcoff="15" dstbuf="o" dstoff="11" cnt="1" depid="-1" deps="-1" hasdep="0"/>
-    </tb>
-  </gpu>
-  <gpu id="6" i_chunks="16" o_chunks="16" s_chunks="0">
-    <tb id="0" send="-1" recv="-1" chan="6">
-      <step s="0" type="cpy" srcbuf="i" srcoff="12" dstbuf="o" dstoff="12" cnt="1" depid="-1" deps="-1" hasdep="0"/>
-    </tb>
-    <tb id="1" send="-1" recv="-1" chan="14">
-      <step s="0" type="cpy" srcbuf="i" srcoff="13" dstbuf="o" dstoff="13" cnt="1" depid="-1" deps="-1" hasdep="0"/>
-    </tb>
-    <tb id="2" send="-1" recv="0" chan="0">
-      <step s="0" type="r" srcbuf="i" srcoff="12" dstbuf="o" dstoff="0" cnt="1" depid="-1" deps="-1" hasdep="0"/>
-    </tb>
-    <tb id="3" send="-1" recv="0" chan="8">
-      <step s="0" type="r" srcbuf="i" srcoff="13" dstbuf="o" dstoff="1" cnt="1" depid="-1" deps="-1" hasdep="0"/>
-    </tb>
-    <tb id="4" send="-1" recv="1" chan="1">
-      <step s="0" type="r" srcbuf="i" srcoff="12" dstbuf="o" dstoff="2" cnt="1" depid="-1" deps="-1" hasdep="0"/>
-    </tb>
-    <tb id="5" send="-1" recv="1" chan="9">
-      <step s="0" type="r" srcbuf="i" srcoff="13" dstbuf="o" dstoff="3" cnt="1" depid="-1" deps="-1" hasdep="0"/>
-    </tb>
-    <tb id="6" send="-1" recv="2" chan="2">
-      <step s="0" type="r" srcbuf="i" srcoff="12" dstbuf="o" dstoff="4" cnt="1" depid="-1" deps="-1" hasdep="0"/>
-    </tb>
-    <tb id="7" send="-1" recv="2" chan="10">
-      <step s="0" type="r" srcbuf="i" srcoff="13" dstbuf="o" dstoff="5" cnt="1" depid="-1" deps="-1" hasdep="0"/>
-    </tb>
-    <tb id="8" send="-1" recv="3" chan="3">
-      <step s="0" type="r" srcbuf="i" srcoff="12" dstbuf="o" dstoff="6" cnt="1" depid="-1" deps="-1" hasdep="0"/>
-    </tb>
-    <tb id="9" send="-1" recv="3" chan="11">
-      <step s="0" type="r" srcbuf="i" srcoff="13" dstbuf="o" dstoff="7" cnt="1" depid="-1" deps="-1" hasdep="0"/>
-    </tb>
-    <tb id="10" send="-1" recv="4" chan="4">
-      <step s="0" type="r" srcbuf="i" srcoff="12" dstbuf="o" dstoff="8" cnt="1" depid="-1" deps="-1" hasdep="0"/>
-    </tb>
-    <tb id="11" send="-1" recv="4" chan="12">
-      <step s="0" type="r" srcbuf="i" srcoff="13" dstbuf="o" dstoff="9" cnt="1" depid="-1" deps="-1" hasdep="0"/>
-    </tb>
-    <tb id="12" send="-1" recv="5" chan="5">
-      <step s="0" type="r" srcbuf="i" srcoff="12" dstbuf="o" dstoff="10" cnt="1" depid="-1" deps="-1" hasdep="0"/>
-    </tb>
-    <tb id="13" send="-1" recv="5" chan="13">
-      <step s="0" type="r" srcbuf="i" srcoff="13" dstbuf="o" dstoff="11" cnt="1" depid="-1" deps="-1" hasdep="0"/>
-    </tb>
-    <tb id="14" send="-1" recv="7" chan="7">
-      <step s="0" type="r" srcbuf="i" srcoff="12" dstbuf="o" dstoff="14" cnt="1" depid="-1" deps="-1" hasdep="0"/>
-    </tb>
-    <tb id="15" send="-1" recv="7" chan="15">
-      <step s="0" type="r" srcbuf="i" srcoff="13" dstbuf="o" dstoff="15" cnt="1" depid="-1" deps="-1" hasdep="0"/>
-    </tb>
-    <tb id="16" send="0" recv="-1" chan="6">
-      <step s="0" type="s" srcbuf="i" srcoff="0" dstbuf="o" dstoff="12" cnt="1" depid="-1" deps="-1" hasdep="0"/>
-    </tb>
-    <tb id="17" send="0" recv="-1" chan="14">
-      <step s="0" type="s" srcbuf="i" srcoff="1" dstbuf="o" dstoff="13" cnt="1" depid="-1" deps="-1" hasdep="0"/>
-    </tb>
-    <tb id="18" send="1" recv="-1" chan="6">
-      <step s="0" type="s" srcbuf="i" srcoff="2" dstbuf="o" dstoff="12" cnt="1" depid="-1" deps="-1" hasdep="0"/>
-    </tb>
-    <tb id="19" send="1" recv="-1" chan="14">
-      <step s="0" type="s" srcbuf="i" srcoff="3" dstbuf="o" dstoff="13" cnt="1" depid="-1" deps="-1" hasdep="0"/>
-    </tb>
-    <tb id="20" send="2" recv="-1" chan="6">
-      <step s="0" type="s" srcbuf="i" srcoff="4" dstbuf="o" dstoff="12" cnt="1" depid="-1" deps="-1" hasdep="0"/>
-    </tb>
-    <tb id="21" send="2" recv="-1" chan="14">
-      <step s="0" type="s" srcbuf="i" srcoff="5" dstbuf="o" dstoff="13" cnt="1" depid="-1" deps="-1" hasdep="0"/>
-    </tb>
-    <tb id="22" send="3" recv="-1" chan="6">
-      <step s="0" type="s" srcbuf="i" srcoff="6" dstbuf="o" dstoff="12" cnt="1" depid="-1" deps="-1" hasdep="0"/>
-    </tb>
-    <tb id="23" send="3" recv="-1" chan="14">
-      <step s="0" type="s" srcbuf="i" srcoff="7" dstbuf="o" dstoff="13" cnt="1" depid="-1" deps="-1" hasdep="0"/>
-    </tb>
-    <tb id="24" send="4" recv="-1" chan="6">
-      <step s="0" type="s" srcbuf="i" srcoff="8" dstbuf="o" dstoff="12" cnt="1" depid="-1" deps="-1" hasdep="0"/>
-    </tb>
-    <tb id="25" send="4" recv="-1" chan="14">
-      <step s="0" type="s" srcbuf="i" srcoff="9" dstbuf="o" dstoff="13" cnt="1" depid="-1" deps="-1" hasdep="0"/>
-    </tb>
-    <tb id="26" send="5" recv="-1" chan="6">
-      <step s="0" type="s" srcbuf="i" srcoff="10" dstbuf="o" dstoff="12" cnt="1" depid="-1" deps="-1" hasdep="0"/>
-    </tb>
-    <tb id="27" send="5" recv="-1" chan="14">
-      <step s="0" type="s" srcbuf="i" srcoff="11" dstbuf="o" dstoff="13" cnt="1" depid="-1" deps="-1" hasdep="0"/>
-    </tb>
-    <tb id="28" send="7" recv="-1" chan="6">
-      <step s="0" type="s" srcbuf="i" srcoff="14" dstbuf="o" dstoff="12" cnt="1" depid="-1" deps="-1" hasdep="0"/>
-    </tb>
-    <tb id="29" send="7" recv="-1" chan="14">
-      <step s="0" type="s" srcbuf="i" srcoff="15" dstbuf="o" dstoff="13" cnt="1" depid="-1" deps="-1" hasdep="0"/>
-    </tb>
-  </gpu>
-  <gpu id="7" i_chunks="16" o_chunks="16" s_chunks="0">
-    <tb id="0" send="-1" recv="-1" chan="7">
-      <step s="0" type="cpy" srcbuf="i" srcoff="14" dstbuf="o" dstoff="14" cnt="1" depid="-1" deps="-1" hasdep="0"/>
-    </tb>
-    <tb id="1" send="-1" recv="-1" chan="15">
-      <step s="0" type="cpy" srcbuf="i" srcoff="15" dstbuf="o" dstoff="15" cnt="1" depid="-1" deps="-1" hasdep="0"/>
-    </tb>
-    <tb id="2" send="-1" recv="0" chan="0">
-      <step s="0" type="r" srcbuf="i" srcoff="14" dstbuf="o" dstoff="0" cnt="1" depid="-1" deps="-1" hasdep="0"/>
-    </tb>
-    <tb id="3" send="-1" recv="0" chan="8">
-      <step s="0" type="r" srcbuf="i" srcoff="15" dstbuf="o" dstoff="1" cnt="1" depid="-1" deps="-1" hasdep="0"/>
-    </tb>
-    <tb id="4" send="-1" recv="1" chan="1">
-      <step s="0" type="r" srcbuf="i" srcoff="14" dstbuf="o" dstoff="2" cnt="1" depid="-1" deps="-1" hasdep="0"/>
-    </tb>
-    <tb id="5" send="-1" recv="1" chan="9">
-      <step s="0" type="r" srcbuf="i" srcoff="15" dstbuf="o" dstoff="3" cnt="1" depid="-1" deps="-1" hasdep="0"/>
-    </tb>
-    <tb id="6" send="-1" recv="2" chan="2">
-      <step s="0" type="r" srcbuf="i" srcoff="14" dstbuf="o" dstoff="4" cnt="1" depid="-1" deps="-1" hasdep="0"/>
-    </tb>
-    <tb id="7" send="-1" recv="2" chan="10">
-      <step s="0" type="r" srcbuf="i" srcoff="15" dstbuf="o" dstoff="5" cnt="1" depid="-1" deps="-1" hasdep="0"/>
-    </tb>
-    <tb id="8" send="-1" recv="3" chan="3">
-      <step s="0" type="r" srcbuf="i" srcoff="14" dstbuf="o" dstoff="6" cnt="1" depid="-1" deps="-1" hasdep="0"/>
-    </tb>
-    <tb id="9" send="-1" recv="3" chan="11">
-      <step s="0" type="r" srcbuf="i" srcoff="15" dstbuf="o" dstoff="7" cnt="1" depid="-1" deps="-1" hasdep="0"/>
-    </tb>
-    <tb id="10" send="-1" recv="4" chan="4">
-      <step s="0" type="r" srcbuf="i" srcoff="14" dstbuf="o" dstoff="8" cnt="1" depid="-1" deps="-1" hasdep="0"/>
-    </tb>
-    <tb id="11" send="-1" recv="4" chan="12">
-      <step s="0" type="r" srcbuf="i" srcoff="15" dstbuf="o" dstoff="9" cnt="1" depid="-1" deps="-1" hasdep="0"/>
-    </tb>
-    <tb id="12" send="-1" recv="5" chan="5">
-      <step s="0" type="r" srcbuf="i" srcoff="14" dstbuf="o" dstoff="10" cnt="1" depid="-1" deps="-1" hasdep="0"/>
-    </tb>
-    <tb id="13" send="-1" recv="5" chan="13">
-      <step s="0" type="r" srcbuf="i" srcoff="15" dstbuf="o" dstoff="11" cnt="1" depid="-1" deps="-1" hasdep="0"/>
-    </tb>
-    <tb id="14" send="-1" recv="6" chan="6">
-      <step s="0" type="r" srcbuf="i" srcoff="14" dstbuf="o" dstoff="12" cnt="1" depid="-1" deps="-1" hasdep="0"/>
-    </tb>
-    <tb id="15" send="-1" recv="6" chan="14">
-      <step s="0" type="r" srcbuf="i" srcoff="15" dstbuf="o" dstoff="13" cnt="1" depid="-1" deps="-1" hasdep="0"/>
-    </tb>
-    <tb id="16" send="0" recv="-1" chan="7">
-      <step s="0" type="s" srcbuf="i" srcoff="0" dstbuf="o" dstoff="14" cnt="1" depid="-1" deps="-1" hasdep="0"/>
-    </tb>
-    <tb id="17" send="0" recv="-1" chan="15">
-      <step s="0" type="s" srcbuf="i" srcoff="1" dstbuf="o" dstoff="15" cnt="1" depid="-1" deps="-1" hasdep="0"/>
-    </tb>
-    <tb id="18" send="1" recv="-1" chan="7">
-      <step s="0" type="s" srcbuf="i" srcoff="2" dstbuf="o" dstoff="14" cnt="1" depid="-1" deps="-1" hasdep="0"/>
-    </tb>
-    <tb id="19" send="1" recv="-1" chan="15">
-      <step s="0" type="s" srcbuf="i" srcoff="3" dstbuf="o" dstoff="15" cnt="1" depid="-1" deps="-1" hasdep="0"/>
-    </tb>
-    <tb id="20" send="2" recv="-1" chan="7">
-      <step s="0" type="s" srcbuf="i" srcoff="4" dstbuf="o" dstoff="14" cnt="1" depid="-1" deps="-1" hasdep="0"/>
-    </tb>
-    <tb id="21" send="2" recv="-1" chan="15">
-      <step s="0" type="s" srcbuf="i" srcoff="5" dstbuf="o" dstoff="15" cnt="1" depid="-1" deps="-1" hasdep="0"/>
-    </tb>
-    <tb id="22" send="3" recv="-1" chan="7">
-      <step s="0" type="s" srcbuf="i" srcoff="6" dstbuf="o" dstoff="14" cnt="1" depid="-1" deps="-1" hasdep="0"/>
-    </tb>
-    <tb id="23" send="3" recv="-1" chan="15">
-      <step s="0" type="s" srcbuf="i" srcoff="7" dstbuf="o" dstoff="15" cnt="1" depid="-1" deps="-1" hasdep="0"/>
-    </tb>
-    <tb id="24" send="4" recv="-1" chan="7">
-      <step s="0" type="s" srcbuf="i" srcoff="8" dstbuf="o" dstoff="14" cnt="1" depid="-1" deps="-1" hasdep="0"/>
-    </tb>
-    <tb id="25" send="4" recv="-1" chan="15">
-      <step s="0" type="s" srcbuf="i" srcoff="9" dstbuf="o" dstoff="15" cnt="1" depid="-1" deps="-1" hasdep="0"/>
-    </tb>
-    <tb id="26" send="5" recv="-1" chan="7">
-      <step s="0" type="s" srcbuf="i" srcoff="10" dstbuf="o" dstoff="14" cnt="1" depid="-1" deps="-1" hasdep="0"/>
-    </tb>
-    <tb id="27" send="5" recv="-1" chan="15">
-      <step s="0" type="s" srcbuf="i" srcoff="11" dstbuf="o" dstoff="15" cnt="1" depid="-1" deps="-1" hasdep="0"/>
-    </tb>
-    <tb id="28" send="6" recv="-1" chan="7">
-      <step s="0" type="s" srcbuf="i" srcoff="12" dstbuf="o" dstoff="14" cnt="1" depid="-1" deps="-1" hasdep="0"/>
-    </tb>
-    <tb id="29" send="6" recv="-1" chan="15">
-      <step s="0" type="s" srcbuf="i" srcoff="13" dstbuf="o" dstoff="15" cnt="1" depid="-1" deps="-1" hasdep="0"/>
-    </tb>
-  </gpu>
-</algo>
diff --git a/tools/msccl-algorithms/alltoall-8n-1mb-4mb.xml b/tools/msccl-algorithms/alltoall-8n-512kb-7mb.xml
similarity index 99%
rename from tools/msccl-algorithms/alltoall-8n-1mb-4mb.xml
rename to tools/msccl-algorithms/alltoall-8n-512kb-7mb.xml
index f2f7076545..9646f270fe 100644
--- a/tools/msccl-algorithms/alltoall-8n-1mb-4mb.xml
+++ b/tools/msccl-algorithms/alltoall-8n-512kb-7mb.xml
@@ -1,6 +1,6 @@
 <!-- Copyright (c) Microsoft Corporation. -->
 <!-- Licensed under the MIT License. -->
-<algo name="alltoall_llm" proto="Simple" nchannels="32" nchunksperloop="32" ngpus="8" coll="alltoall" inplace="0" outofplace="1" minBytes="1048576" maxBytes="4194303">
+<algo name="alltoall_llm" proto="Simple" nchannels="32" nchunksperloop="32" ngpus="8" coll="alltoall" inplace="0" outofplace="1" minBytes="524289" maxBytes="7340032">
   <gpu id="0" i_chunks="32" o_chunks="32" s_chunks="0">
     <tb id="0" send="-1" recv="-1" chan="0">
       <step s="0" type="cpy" srcbuf="i" srcoff="0" dstbuf="o" dstoff="0" cnt="1" depid="-1" deps="-1" hasdep="0"/>
diff --git a/tools/msccl-algorithms/alltoall-8n-4mb-16mb.xml b/tools/msccl-algorithms/alltoall-8n-7mb-43mb.xml
similarity index 99%
rename from tools/msccl-algorithms/alltoall-8n-4mb-16mb.xml
rename to tools/msccl-algorithms/alltoall-8n-7mb-43mb.xml
index f50bced233..fdd6dd0cc3 100644
--- a/tools/msccl-algorithms/alltoall-8n-4mb-16mb.xml
+++ b/tools/msccl-algorithms/alltoall-8n-7mb-43mb.xml
@@ -1,6 +1,6 @@
 <!-- Copyright (c) Microsoft Corporation. -->
 <!-- Licensed under the MIT License. -->
-<algo name="alltoall_llm" proto="Simple" nchannels="8" nchunksperloop="64" ngpus="8" coll="alltoall" inplace="0" outofplace="1" minBytes="4194303" maxBytes="16777216">
+<algo name="alltoall_llm" proto="Simple" nchannels="8" nchunksperloop="64" ngpus="8" coll="alltoall" inplace="0" outofplace="1" minBytes="7340033" maxBytes="45088768">
   <gpu id="0" i_chunks="64" o_chunks="64" s_chunks="0">
     <tb id="0" send="-1" recv="-1" chan="0">
       <step s="0" type="cpy" srcbuf="i" srcoff="0" dstbuf="o" dstoff="0" cnt="1" depid="-1" deps="-1" hasdep="0"/>
diff --git a/tools/msccl-algorithms/alltoall-8n-1kb-4kb.xml b/tools/msccl-algorithms/alltoall-8n-9kb-190kb.xml
similarity index 99%
rename from tools/msccl-algorithms/alltoall-8n-1kb-4kb.xml
rename to tools/msccl-algorithms/alltoall-8n-9kb-190kb.xml
index da81f463b9..d1e0b22d78 100644
--- a/tools/msccl-algorithms/alltoall-8n-1kb-4kb.xml
+++ b/tools/msccl-algorithms/alltoall-8n-9kb-190kb.xml
@@ -1,6 +1,6 @@
 <!-- Copyright (c) Microsoft Corporation. -->
 <!-- Licensed under the MIT License. -->
-<algo name="alltoall_llm" proto="LL" nchannels="2" nchunksperloop="16" ngpus="8" coll="alltoall" inplace="0" outofplace="1" minBytes="1024" maxBytes="4095">
+<algo name="alltoall_llm" proto="LL" nchannels="2" nchunksperloop="16" ngpus="8" coll="alltoall" inplace="0" outofplace="1" minBytes="9217" maxBytes="194560">
   <gpu id="0" i_chunks="16" o_chunks="16" s_chunks="0">
     <tb id="0" send="-1" recv="-1" chan="0">
       <step s="0" type="cpy" srcbuf="i" srcoff="0" dstbuf="o" dstoff="0" cnt="1" depid="-1" deps="-1" hasdep="0"/>