From 7dd5113bdf616d9fad8df36c003c00c75fa4fb2b Mon Sep 17 00:00:00 2001 From: Ben Sander Date: Fri, 17 Feb 2017 17:14:55 -0600 Subject: [PATCH] Add first step to a "registerd" mode in hipBusBandwidth. [ROCm/hip-tests commit: b25691cb87f8bb1010cb37a0973cbe9e5b572986] --- .../hipBusBandwidth/hipBusBandwidth.cpp | 163 +++++++++++------- 1 file changed, 104 insertions(+), 59 deletions(-) diff --git a/projects/hip-tests/samples/1_Utils/hipBusBandwidth/hipBusBandwidth.cpp b/projects/hip-tests/samples/1_Utils/hipBusBandwidth/hipBusBandwidth.cpp index 7cb3e7908e..a1b2fd1705 100644 --- a/projects/hip-tests/samples/1_Utils/hipBusBandwidth/hipBusBandwidth.cpp +++ b/projects/hip-tests/samples/1_Utils/hipBusBandwidth/hipBusBandwidth.cpp @@ -6,9 +6,12 @@ #include "ResultDatabase.h" +enum MallocMode {MallocPinned, MallocUnpinned, MallocRegistered}; + // Cmdline parms: bool p_verbose = false; -bool p_pinned = true; +MallocMode p_malloc_mode = MallocPinned; +int p_numa_ctl = -1; int p_iterations = 10; int p_beatsperiteration=1; int p_device = 0; @@ -21,7 +24,7 @@ bool p_h2d = true; bool p_d2h = true; bool p_bidir = true; - +#define NO_CHECK #define CHECK_HIP_ERROR() \ @@ -36,6 +39,14 @@ bool p_bidir = true; } +std::string mallocModeString(int mallocMode) { + switch (mallocMode) { + case MallocPinned : return "pinned"; + case MallocUnpinned: return "unpinned"; + case MallocRegistered: return "registered"; + default: return "mallocmode-UNKNOWN"; + }; +}; // **************************************************************************** int sizeToBytes(int size) { @@ -106,7 +117,7 @@ void RunBenchmark_H2D(ResultDatabase &resultDB) // Create some host memory pattern float *hostMem = NULL; - if (p_pinned) + if (p_malloc_mode == MallocPinned) { hipHostMalloc((void**)&hostMem, sizeof(float) * numMaxFloats); while (hipGetLastError() != hipSuccess) @@ -116,20 +127,29 @@ void RunBenchmark_H2D(ResultDatabase &resultDB) --nSizes; if (nSizes < 1) { - std::cerr << "Error: Couldn't allocated any pinned buffer\n"; + std::cerr << "Error: Couldn't allocate any pinned buffer\n"; return; } numMaxFloats = 1024 * (sizes[nSizes-1]) / 4; hipHostMalloc((void**)&hostMem, sizeof(float) * numMaxFloats); } } - else + else if (p_malloc_mode == MallocUnpinned) { if (p_alignedhost) { hostMem = (float*)aligned_alloc(p_alignedhost, numMaxFloats*sizeof(float)); } else { hostMem = new float[numMaxFloats]; } + } + else if (p_malloc_mode == MallocRegistered) + { + if (p_numa_ctl == -1) { + hostMem = (float*)malloc(numMaxFloats*sizeof(float)); + } + + hipHostRegister(hostMem, numMaxFloats * sizeof(float), 0); + CHECK_HIP_ERROR(); } for (int i = 0; i < numMaxFloats; i++) @@ -146,7 +166,7 @@ void RunBenchmark_H2D(ResultDatabase &resultDB) --nSizes; if (nSizes < 1) { - std::cerr << "Error: Couldn't allocated any device buffer\n"; + std::cerr << "Error: Couldn't allocate any device buffer\n"; return; } numMaxFloats = 1024 * (sizes[nSizes-1]) / 4; @@ -199,8 +219,8 @@ void RunBenchmark_H2D(ResultDatabase &resultDB) } else { sprintf(sizeStr, "%9s", sizeToString(thisSize).c_str()); } - resultDB.AddResult(std::string("H2D_Bandwidth") + (p_pinned ? "_Pinned" : "_Unpinned"), sizeStr, "GB/sec", speed); - resultDB.AddResult(std::string("H2D_Time") + (p_pinned ? "_Pinned" : "_Unpinned"), sizeStr, "ms", t); + resultDB.AddResult(std::string("H2D_Bandwidth") + "_" + mallocModeString(p_malloc_mode), sizeStr, "GB/sec", speed); + resultDB.AddResult(std::string("H2D_Time") + mallocModeString(p_malloc_mode), sizeStr, "ms", t); if (p_onesize) { break; @@ -212,6 +232,8 @@ void RunBenchmark_H2D(ResultDatabase &resultDB) numMaxFloats = sizeToBytes(p_onesize) / sizeof(float); } +#ifndef NO_CHECK + // Check. First reset the host memory, then copy-back result. Then compare against original ref value. for (int i = 0; i < numMaxFloats; i++) { @@ -225,24 +247,36 @@ void RunBenchmark_H2D(ResultDatabase &resultDB) printf ("error: H2D. i=%d reference:%6.f != copyback:%6.2f\n", i, ref, hostMem[i]); } } +#endif // Cleanup hipFree((void*)device); CHECK_HIP_ERROR(); - if (p_pinned) - { + switch (p_malloc_mode) { + case MallocPinned: hipHostFree((void*)hostMem); CHECK_HIP_ERROR(); - } - else - { + break; + + case MallocUnpinned: if (p_alignedhost) { delete[] hostMem; } else { free(hostMem); } + break; + + case MallocRegistered: + hipHostUnregister(hostMem); + CHECK_HIP_ERROR(); + free(hostMem); + break; + default: + assert(0); } + + hipEventDestroy(start); hipEventDestroy(stop); } @@ -257,38 +291,40 @@ void RunBenchmark_D2H(ResultDatabase &resultDB) // Create some host memory pattern float *hostMem1; float *hostMem2; - if (p_pinned) + if (p_malloc_mode == MallocPinned) { hipHostMalloc((void**)&hostMem1, sizeof(float)*numMaxFloats); hipError_t err1 = hipGetLastError(); hipHostMalloc((void**)&hostMem2, sizeof(float)*numMaxFloats); hipError_t err2 = hipGetLastError(); - while (err1 != hipSuccess || err2 != hipSuccess) - { - // free the first buffer if only the second failed - if (err1 == hipSuccess) - hipHostFree((void*)hostMem1); + while (err1 != hipSuccess || err2 != hipSuccess) + { + // free the first buffer if only the second failed + if (err1 == hipSuccess) + hipHostFree((void*)hostMem1); - // drop the size and try again - if (p_verbose) std::cout << " - dropping size allocating pinned mem\n"; - --nSizes; - if (nSizes < 1) - { - std::cerr << "Error: Couldn't allocated any pinned buffer\n"; - return; - } - numMaxFloats = 1024 * (sizes[nSizes-1]) / 4; - hipHostMalloc((void**)&hostMem1, sizeof(float)*numMaxFloats); - err1 = hipGetLastError(); - hipHostMalloc((void**)&hostMem2, sizeof(float)*numMaxFloats); - err2 = hipGetLastError(); - } - } - else + // drop the size and try again + if (p_verbose) std::cout << " - dropping size allocating pinned mem\n"; + --nSizes; + if (nSizes < 1) + { + std::cerr << "Error: Couldn't allocate any pinned buffer\n"; + return; + } + numMaxFloats = 1024 * (sizes[nSizes-1]) / 4; + hipHostMalloc((void**)&hostMem1, sizeof(float)*numMaxFloats); + err1 = hipGetLastError(); + hipHostMalloc((void**)&hostMem2, sizeof(float)*numMaxFloats); + err2 = hipGetLastError(); + } + } + else if (p_malloc_mode == MallocUnpinned) { hostMem1 = new float[numMaxFloats]; hostMem2 = new float[numMaxFloats]; } + + for (int i=0; i