AI/rocm-systems
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Make tests check for deltas and report bandwidth

Browse Source 
[ROCm/rccl commit: 6c77476cc1]
This commit is contained in:
Sylvain Jeaugey
2016-09-20 11:48:34 -07:00
parent b540c2b5cc
commit 5dd42ff223
6 changed files with 236 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files
projects/rccl/test/single/all_gather_test.cu
+17 -2
View File
@@ -13,6 +13,9 @@
#include "nccl.h"
#include "test_utilities.h"
int errors = 0;
double min_bw = 10000.0;
bool is_reduction = false;
template<typename T>
void RunTest(T** sendbuff, T** recvbuff, const int N, const ncclDataType_t type,
@@ -84,6 +87,9 @@ void RunTest(T** sendbuff, T** recvbuff, const int N, const ncclDataType_t type,
printf(" %7.3f %5.2f %5.2f %7.0le\n", elapsedSec * 1.0E3, algbw, busbw,
maxDelta);
if (maxDelta > deltaMaxValue(type, is_reduction)) errors++;
if (busbw < min_bw) min_bw = busbw;
}
for (int i = 0; i < nDev; ++i) {
@@ -197,7 +203,7 @@ int main(int argc, char* argv[]) {
RunTests<char>(N / sizeof(char), ncclChar, comms, dList);
RunTests<int>(N / sizeof(int), ncclInt, comms, dList);
#if CUDART_VERSION >= 7050
#ifdef CUDA_HAS_HALF
RunTests<half>(N / sizeof(half), ncclHalf, comms, dList);
#endif
RunTests<float>(N / sizeof(float), ncclFloat, comms, dList);
@@ -211,6 +217,15 @@ int main(int argc, char* argv[]) {
ncclCommDestroy(comms[i]);
free(comms);
exit(EXIT_SUCCESS);
char* str = getenv("NCCL_TESTS_MIN_BW");
double check_min_bw = str ? atof(str) : -1;
printf(" Out of bounds values : %d %s\n", errors, errors ? "FAILED" : "OK");
printf(" Min bus bandwidth : %g %s\n", min_bw, check_min_bw == -1 ? "" : (min_bw < check_min_bw ? "FAILED" : "OK"));
printf("\n");
if (errors || min_bw < check_min_bw)
exit(EXIT_FAILURE);
else
exit(EXIT_SUCCESS);
}
projects/rccl/test/single/all_reduce_test.cu
+20 -2
View File
@@ -15,6 +15,9 @@
#include <nvToolsExt.h>
int csv = false;
int errors = 0;
double min_bw = 10000.0;
bool is_reduction = true;
template<typename T>
void RunTest(T** sendbuff, T** recvbuff, const int N, const ncclDataType_t type,
@@ -95,6 +98,9 @@ void RunTest(T** sendbuff, T** recvbuff, const int N, const ncclDataType_t type,
printf((csv)?"%f,%f,%f,%le,":" %7.3f %5.2f %5.2f %7.0le",
elapsedSec * 1.0E3, algbw, busbw, maxDelta);
if (maxDelta > deltaMaxValue(type, is_reduction)) errors++;
if (busbw < min_bw) min_bw = busbw;
nvtxRangePop();
}
@@ -138,6 +144,9 @@ void RunTest(T** sendbuff, T** recvbuff, const int N, const ncclDataType_t type,
printf((csv)?"%f,%f,%f,%le,":" %7.3f %5.2f %5.2f %7.0le\n",
elapsedSec * 1.0E3, algbw, busbw, maxDelta);
if (maxDelta > deltaMaxValue(type, is_reduction)) errors++;
if (busbw < min_bw) min_bw = busbw;
nvtxRangePop();
}
@@ -260,7 +269,7 @@ int main(int argc, char* argv[]) {
RunTests<char>(N / sizeof(char), ncclChar, comms, dList);
RunTests<int>(N / sizeof(int), ncclInt, comms, dList);
#if CUDART_VERSION >= 7050
#ifdef CUDA_HAS_HALF
RunTests<half>(N / sizeof(half), ncclHalf, comms, dList);
#endif
RunTests<float>(N / sizeof(float), ncclFloat, comms, dList);
@@ -274,6 +283,15 @@ int main(int argc, char* argv[]) {
ncclCommDestroy(comms[i]);
free(comms);
exit(EXIT_SUCCESS);
char* str = getenv("NCCL_TESTS_MIN_BW");
double check_min_bw = str ? atof(str) : -1;
printf(" Out of bounds values : %d %s\n", errors, errors ? "FAILED" : "OK");
printf(" Min bus bandwidth : %g %s\n", min_bw, check_min_bw == -1 ? "" : (min_bw < check_min_bw ? "FAILED" : "OK"));
printf("\n");
if (errors || min_bw < check_min_bw)
exit(EXIT_FAILURE);
else
exit(EXIT_SUCCESS);
}
projects/rccl/test/single/broadcast_test.cu
+16 -1
View File
@@ -13,6 +13,9 @@
#include "nccl.h"
#include "test_utilities.h"
int errors = 0;
double min_bw = 10000.0;
bool is_reduction = false;
template<typename T>
void RunTest(T** buff, const int N, const ncclDataType_t type, const int root,
@@ -86,6 +89,9 @@ void RunTest(T** buff, const int N, const ncclDataType_t type, const int root,
printf(" %7.3f %5.2f %5.2f %7.0le\n", elapsedSec * 1.0E3, algbw, busbw,
maxDelta);
if (maxDelta > deltaMaxValue(type, is_reduction)) errors++;
if (busbw < min_bw) min_bw = busbw;
}
for(int i=0; i < nDev; ++i) {
@@ -211,6 +217,15 @@ int main(int argc, char* argv[]) {
ncclCommDestroy(comms[i]);
free(comms);
exit(EXIT_SUCCESS);
char* str = getenv("NCCL_TESTS_MIN_BW");
double check_min_bw = str ? atof(str) : -1;
printf(" Out of bounds values : %d %s\n", errors, errors ? "FAILED" : "OK");
printf(" Min bus bandwidth : %g %s\n", min_bw, check_min_bw == -1 ? "" : (min_bw < check_min_bw ? "FAILED" : "OK"));
printf("\n");
if (errors || min_bw < check_min_bw)
exit(EXIT_FAILURE);
else
exit(EXIT_SUCCESS);
}
projects/rccl/test/single/reduce_scatter_test.cu
+19 -1
View File
@@ -13,6 +13,9 @@
#include "nccl.h"
#include "test_utilities.h"
int errors = 0;
double min_bw = 10000.0;
bool is_reduction = true;
template<typename T>
void RunTest(T** sendbuff, T** recvbuff, const int N, const ncclDataType_t type,
@@ -90,6 +93,9 @@ void RunTest(T** sendbuff, T** recvbuff, const int N, const ncclDataType_t type,
printf(" %7.3f %5.2f %5.2f %7.0le", elapsedSec * 1.0E3, algbw, busbw,
maxDelta);
if (maxDelta > deltaMaxValue(type, is_reduction)) errors++;
if (busbw < min_bw) min_bw = busbw;
}
{
@@ -126,6 +132,9 @@ void RunTest(T** sendbuff, T** recvbuff, const int N, const ncclDataType_t type,
printf(" %7.3f %5.2f %5.2f %7.0le\n", elapsedSec * 1.0E3, algbw, busbw,
maxDelta);
if (maxDelta > deltaMaxValue(type, is_reduction)) errors++;
if (busbw < min_bw) min_bw = busbw;
}
for (int i = 0; i < nDev; ++i) {
@@ -258,6 +267,15 @@ int main(int argc, char* argv[]) {
ncclCommDestroy(comms[i]);
free(comms);
exit(EXIT_SUCCESS);
char* str = getenv("NCCL_TESTS_MIN_BW");
double check_min_bw = str ? atof(str) : -1;
printf(" Out of bounds values : %d %s\n", errors, errors ? "FAILED" : "OK");
printf(" Min bus bandwidth : %g %s\n", min_bw, check_min_bw == -1 ? "" : (min_bw < check_min_bw ? "FAILED" : "OK"));
printf("\n");
if (errors || min_bw < check_min_bw)
exit(EXIT_FAILURE);
else
exit(EXIT_SUCCESS);
}
projects/rccl/test/single/reduce_test.cu
+19 -1
View File
@@ -15,6 +15,9 @@
#include <nvToolsExt.h>
int csv = false;
int errors = 0;
double min_bw = 10000.0;
bool is_reduction = true;
template<typename T>
void RunTest(T** sendbuff, T** recvbuff, const int N, const ncclDataType_t type,
@@ -94,6 +97,9 @@ void RunTest(T** sendbuff, T** recvbuff, const int N, const ncclDataType_t type,
printf((csv)?"%f,%f,%f,%le,":" %7.3f %5.2f %5.2f %7.0le",
elapsedSec * 1.0E3, algbw, busbw, maxDelta);
if (maxDelta > deltaMaxValue(type, is_reduction)) errors++;
if (busbw < min_bw) min_bw = busbw;
nvtxRangePop();
}
@@ -133,6 +139,9 @@ void RunTest(T** sendbuff, T** recvbuff, const int N, const ncclDataType_t type,
printf((csv)?"%f,%f,%f,%le,":" %7.3f %5.2f %5.2f %7.0le\n",
elapsedSec * 1.0E3, algbw, busbw, maxDelta);
if (maxDelta > deltaMaxValue(type, is_reduction)) errors++;
if (busbw < min_bw) min_bw = busbw;
nvtxRangePop();
}
@@ -272,6 +281,15 @@ int main(int argc, char* argv[]) {
ncclCommDestroy(comms[i]);
free(comms);
exit(EXIT_SUCCESS);
char* str = getenv("NCCL_TESTS_MIN_BW");
double check_min_bw = str ? atof(str) : -1;
printf(" Out of bounds values : %d %s\n", errors, errors ? "FAILED" : "OK");
printf(" Min bus bandwidth : %g %s\n", min_bw, check_min_bw == -1 ? "" : (min_bw < check_min_bw ? "FAILED" : "OK"));
printf("\n");
if (errors || min_bw < check_min_bw)
exit(EXIT_FAILURE);
else
exit(EXIT_SUCCESS);
}