diff --git a/src/alltoallv.cu b/src/alltoallv.cu index 34b6462619..bb335eda87 100644 --- a/src/alltoallv.cu +++ b/src/alltoallv.cu @@ -181,7 +181,7 @@ testResult_t AlltoAllvRunTest(struct threadArgs* args, int root, ncclDataType_t } for (int i=0; ifirst; - if (std::next(iter) != outputValuesKeys.end()) { - _out << ", "; + _outputData.push_back(outputValuesKeys); +} + +void Reporter::writeFile() { + if (!isMainThread() || !_outputValid) + return; + + if (_outputFormat == "csv") { + _out << "numCycle,"; + _out << "collective,"; +#ifdef MPI_SUPPORT + _out << "ranks,rankspernode,gpusperrank,"; +#else + _out << "gpus,"; +#endif + _out << "size,type,redop,inplace,time,algbw,busbw,#wrong\n"; + for (auto iterEntries = _outputData.begin(); iterEntries != _outputData.end(); ++iterEntries) { + for (auto iterVals = (*iterEntries).begin(); iterVals != (*iterEntries).end(); ++iterVals) { + _out << iterVals->first; + if (std::next(iterVals) != (*iterEntries).end()) { + _out << ","; + } } - } else { //json - if (iter == outputValuesKeys.begin()) { - _out << "{"; + _out << std::endl; + } + } else { //json + _out << "[" << std::endl; + for (auto iterEntries = _outputData.begin(); iterEntries != _outputData.end(); ++iterEntries) { + for (auto iterVals = (*iterEntries).begin(); iterVals != (*iterEntries).end(); ++iterVals) { + if (iterVals == (*iterEntries).begin()) { + _out << "{"; + } + _out << "\"" << iterVals->second << "\":" << iterVals->first; + if (std::next(iterVals) != (*iterEntries).end()) { + _out << ", "; + } } - _out << "\"" << iter->second << "\":" << iter->first; - if (std::next(iter) != outputValuesKeys.end()) { - _out << ", "; + if (std::next(iterEntries) != _outputData.end()) { + _out << "}," << std::endl; } else { - _out << "}"; + _out << "}" << std::endl; } } + _out << "]" << std::endl; } - _out << std::endl; } + bool Reporter::isMainThread() { return is_main_thread == 1; } #define NUM_BLOCKS 32 @@ -1711,6 +1728,8 @@ testResult_t run() { MPI_Finalize(); #endif + reporter.writeFile(); + // 'cuda-memcheck --leak-check full' requires this PRINT("%s\n", ncclGetLastError(NULL)); cudaDeviceReset(); diff --git a/src/common.h b/src/common.h index 7a961c15a5..97ea1829f5 100644 --- a/src/common.h +++ b/src/common.h @@ -22,6 +22,8 @@ #include #include #include +#include +#include // Ensures backward compatibility for FP8 datatypes #if NCCL_VERSION_CODE < NCCL_VERSION(2,24,3) @@ -119,6 +121,7 @@ class Reporter { ~Reporter() { if (_outputValid) { _out.close(); } }; void setParameters(const size_t numCycle, const char* name, const char* typeName, const char* opName); void addResult(int gpusPerRank, int ranksPerNode, int totalRanks, size_t numBytes, int inPlace, double timeUsec, double algBw, double busBw, int64_t wrongElts = -1); + void writeFile(); private: bool isMainThread(); @@ -132,6 +135,7 @@ class Reporter { std::string _collectiveName; std::string _typeName; std::string _opName; + std::vector>> _outputData; }; struct testEngine { diff --git a/src/hypercube.cu b/src/hypercube.cu index ff9201045f..f5d94f026d 100644 --- a/src/hypercube.cu +++ b/src/hypercube.cu @@ -101,7 +101,7 @@ testResult_t HyperCubeRunTest(struct threadArgs* args, int root, ncclDataType_t int nRanks = args->nProcs*args->nThreads*args->nGpus; if (nRanks && !(nRanks & (nRanks - 1))) { for (int i=0; i