diff --git a/tools/TopoVisual/4_nodes.log.png b/tools/TopoVisual/4_nodes.log.png new file mode 100644 index 0000000000..45f34bc1dd Binary files /dev/null and b/tools/TopoVisual/4_nodes.log.png differ diff --git a/tools/TopoVisual/README.md b/tools/TopoVisual/README.md new file mode 100644 index 0000000000..5edc251861 --- /dev/null +++ b/tools/TopoVisual/README.md @@ -0,0 +1,35 @@ +# Topology Visualizer +Topology Visualizer extracts topology information from RCCL log file and presents graphically. Less than optimal connections between GPUs and nodes are highlighted in red for easy identification. + +## Requirements +Following packages are required to run Topology Visualizer: +1. gawk +2. graphviz + +## Usage +Topology Visualizer accepts both RCCL log files or simulator output, i.e. [Topology Explorer](https://github.com/ROCmSoftwarePlatform/rccl/tree/master/tools/topo_expl "Topology Explorer"). + +RCCL logs needs to be collected with NCCL_DEBUG=INFO and NCCL_DEBUG_SUBSYS=INIT,GRAPH environmental variables. Example command line: +```shell +mpirun -np 4 -host rocm-framework-1,rocm-framework-3,rocm-framework-5,rocm-framework-6 \ + -env HSA_FORCE_FINE_GRAIN_PCIE 1 -env NCCL_DEBUG INFO -env NCCL_DEBUG_SUBSYS INIT,GRAPH \ + ~/rccl-tests/build/all_reduce_perf -b 8 -e 128M -f 2 -g 8 | tee ~/4_nodes.log + +./topo_visual.sh -i 4_nodes.log +``` + +## Legend + +Solid lines: connections over P2P or shared memory + +Dashed lines: connections over network + +Green: P2P connections, network connections with GPU RDMA + +Red: Connections over shared memory or without GPU RDMA + +## Example Output +![image info](./4_nodes.log.png) + +## Copyright +All source code and accompanying documentation are copyright (c) 2019-2020 Advanced Micro Devices, Inc. All rights reserved. diff --git a/tools/TopoVisual/extract_topo.awk b/tools/TopoVisual/extract_topo.awk new file mode 100755 index 0000000000..e7155f8fe6 --- /dev/null +++ b/tools/TopoVisual/extract_topo.awk @@ -0,0 +1,172 @@ +#!/usr/bin/gawk -f +# Copyright (c) 2019-2020 Advanced Micro Devices, Inc. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE.# Usage: + +BEGIN { + max_rank=0 + rings[""]=0 + max_ring=0 + treedns[""]=0 + max_treedn=0 + conn[""]=0 + col_start=2 + col_p1=col_start+1 + col_p3=col_start+3 + col_p4=col_start+4 + col_p5=col_start+5 + col_p6=col_start+6 + col_p7=col_start+7 + col_p8=col_start+8 +} + +{ + if($3=="NCCL" && $4=="INFO" && col_start==2) { + col_start=5 + col_p1=col_start+1 + col_p3=col_start+3 + col_p4=col_start+4 + col_p5=col_start+5 + col_p6=col_start+6 + col_p7=col_start+7 + col_p8=col_start+8 + } + + if($col_start=="Ring" && $col_p4="->" && $col_p6=="->") { + chan=strtonum($col_p1) + rank=strtonum($col_p5) + next_rank=strtonum($col_p7) + rings[rank "," next_rank "," chan]="1" + if(chan>max_ring) + max_ring=chan + if(rank>max_rank) + max_rank=rank + } + + if($col_start=="Trees") { + col_1=col_start+1 + col_2=col_start+2 + do { + match($col_1, /\[([0-9]+)\]/, ary) + chan=strtonum(ary[1]) + match($col_2, /(\-?[0-9]+)\/(\-?[0-9]+)\/(\-?[0-9]+)\->(\-?[0-9]+)\->(\-?[0-9]+)\|(\-?[0-9]+)\->(\-?[0-9]+)\->(\-?[0-9]+)\/(\-?[0-9]+)\/(\-?[0-9]+)/, ary) + if(ary[8]!="-1") + treedns[ary[7] "," ary[8] "," chan]="1" + if(ary[9]!="-1") + treedns[ary[7] "," ary[9] "," chan]="1" + if(ary[10]!="-1") + treedns[ary[7] "," ary[10] "," chan]="1" + if(chan>max_treedn) + max_treedn=chan + col_1=col_1+2 + col_2=col_2+2 + } while ($col_1!="") + } + + if($col_p6=="via") { + match($col_p1, /([0-9]+)/, ary) + chan=strtonum(ary[1]) + match($col_p3, /([0-9]+)\[.*\]/, ary) + s=ary[1] + match($col_p5, /([0-9]+)\[.*\]/, ary) + d=ary[1] + conn[s "," d "," chan]=$col_p7 + } + + if($col_p6=="[receive]" && $col_p7=="via") { + match($col_p1, /([0-9]+)/, ary) + chan=strtonum(ary[1]) + match($col_p3, /([0-9]+)\[.*\]/, ary) + s=ary[1] + match($col_p5, /([0-9]+)\[.*\]/, ary) + d=ary[1] + conn[s "," d "," chan]=$col_p8 + } +} + +END { + printf "digraph RCCL {\n" + for(r=0;r t%d_%d [label=\"%s\",color=\"%s\",style=\"%s\",fontname=\"Helvetica\"];\n", r, s, r, d, val, color, style + } + } + } + printf "\n" + for(s=0;s<=max_rank;s++) { + printf " t%d_%d [label=\"%d\",fontsize=\"28\"];\n", r, s, s + } + printf " }\n\n" + } + + for(r=0;r r%d_%d [label=\"%s\",color=\"%s\",style=\"%s\",fontname=\"Helvetica\"];\n", r, s, r, d, val, color, style + } + } + } + printf "\n" + for(s=0;s<=max_rank;s++) { + printf " r%d_%d [label=\"%d\",fontsize=\"28\"];\n", r, s, s + } + printf " }\n\n" + } + printf "}\n" +} diff --git a/tools/TopoVisual/topo_visual.sh b/tools/TopoVisual/topo_visual.sh new file mode 100755 index 0000000000..6b77008c55 --- /dev/null +++ b/tools/TopoVisual/topo_visual.sh @@ -0,0 +1,52 @@ +#!/bin/bash +# Copyright (c) 2019-2020 Advanced Micro Devices, Inc. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. + +DIR="$(cd -P "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +exit_error() { + echo "Usage: $0 [ -i input_filename ]" + exit 1 +} + +while getopts ":i:o:" options; do + case "${options}" in + i) + INPUT_NAME=${OPTARG} + ;; + :) + echo "Error: -${OPTARG} requires an argument." + exit_error + ;; + ?) + exit_error + ;; + esac +done + +if [ -z "$INPUT_NAME" ] +then + exit_error +else + $DIR/extract_topo.awk $INPUT_NAME | dot -Tpng -o "$INPUT_NAME.png" + echo "Extracted topology from $INPUT_NAME to $INPUT_NAME.png" +fi + +exit 0 diff --git a/tools/scripts/topo_val.sh b/tools/scripts/topo_val.sh new file mode 100755 index 0000000000..0f97189bd9 --- /dev/null +++ b/tools/scripts/topo_val.sh @@ -0,0 +1,28 @@ +#!/bin/bash +# Copyright (c) 2019-2020 Advanced Micro Devices, Inc. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. + +DIR="$(cd -P "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +for i in {0..9} +do + $DIR/../topo_expl/topo_expl -m $i > "topo_m$i.log" + $DIR/../TopoVisual/topo_visual.sh -i "topo_m$i.log" +done