Transfer files from RAD repository
This commit is contained in:
Brandon Potter
@@ -0,0 +1,70 @@
|
||||
#include <chrono>
|
||||
#include <iostream>
|
||||
#include <stdio.h>
|
||||
#include <mpi.h>
|
||||
#include <unistd.h>
|
||||
#include <hip/hip_runtime.h>
|
||||
using namespace std;
|
||||
|
||||
#define TIME_NOW std::chrono::steady_clock::now()
|
||||
#define TIME_DIFF(a, b) std::chrono::duration_cast<std::chrono::nanoseconds>(a - b).count()
|
||||
|
||||
#define HIPCHECK(cmd) do { \
|
||||
hipError_t e = cmd; \
|
||||
if( e != hipSuccess ) { \
|
||||
printf("Failed: Hip error %s:%d '%s'\n", \
|
||||
__FILE__,__LINE__,hipGetErrorString(e)); \
|
||||
exit(EXIT_FAILURE); \
|
||||
} \
|
||||
} while(0)
|
||||
|
||||
|
||||
#define NCCLCHECK(cmd) do { \
|
||||
ncclResult_t r = cmd; \
|
||||
if (r!= ncclSuccess) { \
|
||||
printf("Failed, NCCL error %s:%d '%s'\n", \
|
||||
__FILE__,__LINE__,ncclGetErrorString(r)); \
|
||||
exit(EXIT_FAILURE); \
|
||||
} \
|
||||
} while(0)
|
||||
|
||||
// Copied from rccl-tests, used to hash hostname
|
||||
static uint64_t getHash(const char* string, size_t n) {
|
||||
// Based on DJB2a, result = result * 33 ^ char
|
||||
uint64_t result = 5381;
|
||||
for (size_t c = 0; c < n; c++) {
|
||||
result = ((result << 5) + result) ^ string[c];
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
/* Generate a hash of the unique identifying string for this host
|
||||
* that will be unique for both bare-metal and container instances
|
||||
* Equivalent of a hash of;
|
||||
*
|
||||
* $(hostname)$(cat /proc/sys/kernel/random/boot_id)
|
||||
*
|
||||
*/
|
||||
#define HOSTID_FILE "/proc/sys/kernel/random/boot_id"
|
||||
static uint64_t getHostHash(const char* hostname) {
|
||||
char hostHash[1024];
|
||||
|
||||
// Fall back is the hostname if something fails
|
||||
(void) strncpy(hostHash, hostname, sizeof(hostHash));
|
||||
int offset = strlen(hostHash);
|
||||
|
||||
FILE *file = fopen(HOSTID_FILE, "r");
|
||||
if (file != NULL) {
|
||||
char *p;
|
||||
if (fscanf(file, "%ms", &p) == 1) {
|
||||
strncpy(hostHash+offset, p, sizeof(hostHash)-offset-1);
|
||||
free(p);
|
||||
}
|
||||
}
|
||||
fclose(file);
|
||||
|
||||
// Make sure the string is terminated
|
||||
hostHash[sizeof(hostHash)-1]='\0';
|
||||
|
||||
return getHash(hostHash, strlen(hostHash));
|
||||
}
|
||||
@@ -0,0 +1,231 @@
|
||||
/*************************************************************************
|
||||
* *
|
||||
* N A S P A R A L L E L B E N C H M A R K S 3.3 *
|
||||
* *
|
||||
* I S *
|
||||
* *
|
||||
*************************************************************************
|
||||
* *
|
||||
* This benchmark is part of the NAS Parallel Benchmark 3.3 suite. *
|
||||
* It is described in NAS Technical Report 95-020. *
|
||||
* *
|
||||
* Permission to use, copy, distribute and modify this software *
|
||||
* for any purpose with or without fee is hereby granted. We *
|
||||
* request, however, that all derived work reference the NAS *
|
||||
* Parallel Benchmarks 3.3. This software is provided "as is" *
|
||||
* without express or implied warranty. *
|
||||
* *
|
||||
* Information on NPB 3.3, including the technical report, the *
|
||||
* original specifications, source code, results and information *
|
||||
* on how to submit new results, is available at: *
|
||||
* *
|
||||
* http://www.nas.nasa.gov/Software/NPB *
|
||||
* *
|
||||
* Send comments or suggestions to npb@nas.nasa.gov *
|
||||
* Send bug reports to npb-bugs@nas.nasa.gov *
|
||||
* *
|
||||
* NAS Parallel Benchmarks Group *
|
||||
* NASA Ames Research Center *
|
||||
* Mail Stop: T27A-1 *
|
||||
* Moffett Field, CA 94035-1000 *
|
||||
* *
|
||||
* E-mail: npb@nas.nasa.gov *
|
||||
* Fax: (650) 604-3957 *
|
||||
* *
|
||||
*************************************************************************
|
||||
* *
|
||||
* Author: M. Yarrow *
|
||||
* H. Jin *
|
||||
* *
|
||||
*************************************************************************/
|
||||
|
||||
#define NUM_WGS 1
|
||||
#define WG_SIZE 1024
|
||||
#define MAX_PES 128
|
||||
|
||||
#define MAX_KEY (1 << 11)
|
||||
|
||||
/*
|
||||
* FUNCTION RANDLC (X, A)
|
||||
*
|
||||
* This routine returns a uniform pseudorandom double precision number in the
|
||||
* range (0, 1) by using the linear congruential generator
|
||||
*
|
||||
* x_{k+1} = a x_k (mod 2^46)
|
||||
*
|
||||
* where 0 < x_k < 2^46 and 0 < a < 2^46. This scheme generates 2^44 numbers
|
||||
* before repeating. The argument A is the same as 'a' in the above formula,
|
||||
* and X is the same as x_0. A and X must be odd double precision integers
|
||||
* in the range (1, 2^46). The returned value RANDLC is normalized to be
|
||||
* between 0 and 1, i.e. RANDLC = 2^(-46) * x_1. X is updated to contain
|
||||
* the new seed x_1, so that subsequent calls to RANDLC using the same
|
||||
* arguments will generate a continuous sequence.
|
||||
*
|
||||
* This routine should produce the same results on any computer with at least
|
||||
* 48 mantissa bits in double precision floating point data. On Cray systems,
|
||||
* double precision should be disabled.
|
||||
*
|
||||
* David H. Bailey October 26, 1990
|
||||
*
|
||||
* IMPLICIT DOUBLE PRECISION (A-H, O-Z)
|
||||
* SAVE KS, R23, R46, T23, T46
|
||||
* DATA KS/0/
|
||||
*
|
||||
* If this is the first call to RANDLC, compute R23 = 2 ^ -23, R46 = 2 ^ -46,
|
||||
* T23 = 2 ^ 23, and T46 = 2 ^ 46. These are computed in loops, rather than
|
||||
* by merely using the ** operator, in order to insure that the results are
|
||||
* exact on all systems. This code assumes that 0.5D0 is represented exactly.
|
||||
*/
|
||||
|
||||
|
||||
|
||||
/*****************************************************************/
|
||||
/************* R A N D L C ************/
|
||||
/************* ************/
|
||||
/************* portable random number generator ************/
|
||||
/*****************************************************************/
|
||||
|
||||
double randlc( double *X, double *A )
|
||||
{
|
||||
static int KS=0;
|
||||
static double R23, R46, T23, T46;
|
||||
double T1, T2, T3, T4;
|
||||
double A1;
|
||||
double A2;
|
||||
double X1;
|
||||
double X2;
|
||||
double Z;
|
||||
int i, j;
|
||||
|
||||
if (KS == 0)
|
||||
{
|
||||
R23 = 1.0;
|
||||
R46 = 1.0;
|
||||
T23 = 1.0;
|
||||
T46 = 1.0;
|
||||
|
||||
for (i=1; i<=23; i++)
|
||||
{
|
||||
R23 = 0.50 * R23;
|
||||
T23 = 2.0 * T23;
|
||||
}
|
||||
for (i=1; i<=46; i++)
|
||||
{
|
||||
R46 = 0.50 * R46;
|
||||
T46 = 2.0 * T46;
|
||||
}
|
||||
KS = 1;
|
||||
}
|
||||
|
||||
/* Break A into two parts such that A = 2^23 * A1 + A2 and set X = N. */
|
||||
|
||||
T1 = R23 * *A;
|
||||
j = T1;
|
||||
A1 = j;
|
||||
A2 = *A - T23 * A1;
|
||||
|
||||
/* Break X into two parts such that X = 2^23 * X1 + X2, compute
|
||||
Z = A1 * X2 + A2 * X1 (mod 2^23), and then
|
||||
X = 2^23 * Z + A2 * X2 (mod 2^46). */
|
||||
|
||||
T1 = R23 * *X;
|
||||
j = T1;
|
||||
X1 = j;
|
||||
X2 = *X - T23 * X1;
|
||||
T1 = A1 * X2 + A2 * X1;
|
||||
|
||||
j = R23 * T1;
|
||||
T2 = j;
|
||||
Z = T1 - T23 * T2;
|
||||
T3 = T23 * Z + A2 * X2;
|
||||
j = R46 * T3;
|
||||
T4 = j;
|
||||
*X = T3 - T46 * T4;
|
||||
return(R46 * *X);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*****************************************************************/
|
||||
/************ F I N D _ M Y _ S E E D ************/
|
||||
/************ ************/
|
||||
/************ returns parallel random number seq seed ************/
|
||||
/*****************************************************************/
|
||||
|
||||
/*
|
||||
* Create a random number sequence of total length nn residing
|
||||
* on np number of processors. Each processor will therefore have a
|
||||
* subsequence of length nn/np. This routine returns that random
|
||||
* number which is the first random number for the subsequence belonging
|
||||
* to processor rank kn, and which is used as seed for proc kn ran # gen.
|
||||
*/
|
||||
|
||||
double find_my_seed( int kn, /* my processor rank, 0<=kn<=num procs */
|
||||
int np, /* np = num procs */
|
||||
long nn, /* total num of ran numbers, all procs */
|
||||
double s, /* Ran num seed, for ex.: 314159265.00 */
|
||||
double a ) /* Ran num gen mult, try 1220703125.00 */
|
||||
{
|
||||
|
||||
long i;
|
||||
|
||||
double t1,t2,t3,an;
|
||||
long mq,nq,kk,ik;
|
||||
|
||||
|
||||
|
||||
nq = nn / np;
|
||||
|
||||
for( mq=0; nq>1; mq++,nq/=2 )
|
||||
;
|
||||
|
||||
t1 = a;
|
||||
|
||||
for( i=1; i<=mq; i++ )
|
||||
t2 = randlc( &t1, &t1 );
|
||||
|
||||
an = t1;
|
||||
|
||||
kk = kn;
|
||||
t1 = s;
|
||||
t2 = an;
|
||||
|
||||
for( i=1; i<=100; i++ )
|
||||
{
|
||||
ik = kk / 2;
|
||||
if( 2 * ik != kk )
|
||||
t3 = randlc( &t1, &t2 );
|
||||
if( ik == 0 )
|
||||
break;
|
||||
t3 = randlc( &t2, &t2 );
|
||||
kk = ik;
|
||||
}
|
||||
|
||||
return( t1 );
|
||||
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
/*****************************************************************/
|
||||
/************* C R E A T E _ S E Q ************/
|
||||
/*****************************************************************/
|
||||
|
||||
void create_seq( double seed, double a, int *key_array, int size )
|
||||
{
|
||||
double x;
|
||||
int i, k;
|
||||
|
||||
k = MAX_KEY/4;
|
||||
|
||||
for (i=0; i < size; i++)
|
||||
{
|
||||
x = randlc(&seed, &a);
|
||||
x += randlc(&seed, &a);
|
||||
x += randlc(&seed, &a);
|
||||
x += randlc(&seed, &a);
|
||||
|
||||
key_array[i] = k*x;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,380 @@
|
||||
#include "mpi.h"
|
||||
#include "common.h"
|
||||
#include "sort.h"
|
||||
|
||||
//#define TIME_PERF
|
||||
#ifdef TIME_PERF
|
||||
#define TIMERS 10
|
||||
__device__ uint64_t timers[TIMERS] = {0};
|
||||
__device__ uint64_t time_start;
|
||||
#define TIMERS_START() \
|
||||
if(threadIdx.x == 0) {\
|
||||
time_start = roc_shmem_timer();\
|
||||
}
|
||||
|
||||
#define TIME(TIMER_NUM) \
|
||||
if(threadIdx.x == 0) {\
|
||||
timers[TIMER_NUM] = roc_shmem_timer() - time_start;\
|
||||
time_start = roc_shmem_timer();\
|
||||
}
|
||||
|
||||
#define OUTPUT_TIME() \
|
||||
if(threadIdx.x == 0 && my_pe == 0) { \
|
||||
uint64_t sum = 0; \
|
||||
for(int i = 0; i < TIMERS; ++i) { \
|
||||
sum += timers[i]; \
|
||||
} \
|
||||
for(int i = 0; i < TIMERS; ++i) { \
|
||||
printf("%d: %f\n", i, (double)timers[i] / (double)sum); \
|
||||
} \
|
||||
}
|
||||
#else
|
||||
#define TIMERS_START()
|
||||
#define TIME(x)
|
||||
#define OUTPUT_TIME()
|
||||
#endif
|
||||
|
||||
__global__ void sort1(volatile int *keys, int *keyBuffer1,
|
||||
int *keyBuffer2, int *sendCount,
|
||||
int *recvCount, int *sendOffset,
|
||||
int *recvOffset, int *outputKeys,
|
||||
size_t size, int n_pes, int my_pe) {
|
||||
__shared__ int bucketCounter[MAX_PES];
|
||||
__shared__ int bucketPtr[MAX_PES];
|
||||
__shared__ int total_size;
|
||||
|
||||
int buckets = n_pes;
|
||||
|
||||
int tid = threadIdx.x; // + blockDim.x * blockIdx.x;
|
||||
const int K_PER_BUCK = (MAX_KEY / buckets);
|
||||
|
||||
// Reset
|
||||
for(int i = threadIdx.x; i < buckets; i += blockDim.x) {
|
||||
bucketCounter[i] = 0;
|
||||
bucketPtr[i] = 0;
|
||||
}
|
||||
__syncthreads();
|
||||
TIMERS_START()
|
||||
// Count size of each bucket
|
||||
for(int i = tid; i < size; i += blockDim.x) {
|
||||
atomicAdd(&bucketCounter[keys[i] / K_PER_BUCK], 1);
|
||||
}
|
||||
__syncthreads();
|
||||
TIME(0)
|
||||
// Update in global memory
|
||||
for(int i = tid; i < buckets; i += blockDim.x) {
|
||||
sendCount[i] = bucketPtr[i] = bucketCounter[i];
|
||||
}
|
||||
__syncthreads();
|
||||
TIME(1)
|
||||
// Perform local scan to get ptrs set
|
||||
for(int shift = 1; shift < buckets; shift *= 2) {
|
||||
int temp = 0;
|
||||
if(threadIdx.x >= shift && threadIdx.x < buckets) {
|
||||
temp = bucketPtr[threadIdx.x - shift];
|
||||
}
|
||||
__syncthreads();
|
||||
if(threadIdx.x < buckets) {
|
||||
bucketPtr[threadIdx.x] += temp;
|
||||
}
|
||||
__syncthreads();
|
||||
}
|
||||
__syncthreads();
|
||||
TIME(2)
|
||||
// Find offsets of where we're sending
|
||||
for(int i = threadIdx.x; i < buckets; i += blockDim.x) {
|
||||
sendOffset[i] = bucketPtr[i] - sendCount[i];
|
||||
}
|
||||
// Sort keys into buckets
|
||||
for(int i = threadIdx.x; i < size; i += blockDim.x) {
|
||||
int loc = atomicAdd(&bucketPtr[keys[i] / K_PER_BUCK], -1) - 1;
|
||||
keyBuffer1[loc] = keys[i];
|
||||
}
|
||||
TIME(3)
|
||||
OUTPUT_TIME()
|
||||
}
|
||||
|
||||
__global__ void sort2(volatile int *keys, int *keyBuffer1,
|
||||
int *keyBuffer2, int *sendCount,
|
||||
int *recvCount, int *sendOffset,
|
||||
int *recvOffset, int *outputKeys,
|
||||
size_t size, int n_pes, int my_pe) {
|
||||
__shared__ int total_size;
|
||||
|
||||
int buckets = n_pes;
|
||||
|
||||
int tid = threadIdx.x; // + blockDim.x * blockIdx.x;
|
||||
const int K_PER_BUCK = (MAX_KEY / buckets);
|
||||
|
||||
for(int i = threadIdx.x; i < K_PER_BUCK; i += blockDim.x)
|
||||
outputKeys[i] = 0;
|
||||
__syncthreads();
|
||||
TIME(5)
|
||||
int min_key_val = my_pe * K_PER_BUCK;
|
||||
int max_key_val = (my_pe + 1) * K_PER_BUCK - 1;
|
||||
|
||||
int *key_buff_ptr = outputKeys - min_key_val;
|
||||
for(int i = threadIdx.x; i < total_size; i += blockDim.x) {
|
||||
atomicAdd(&key_buff_ptr[keyBuffer2[i]], 1);
|
||||
}
|
||||
__syncthreads();
|
||||
TIME(6)
|
||||
// Perform local scan on keys
|
||||
for(int shift = 1; shift < K_PER_BUCK; shift *= 2) {
|
||||
int temp = 0;
|
||||
if(threadIdx.x >= shift && threadIdx.x < K_PER_BUCK) {
|
||||
temp = outputKeys[threadIdx.x - shift];
|
||||
}
|
||||
__syncthreads();
|
||||
if(threadIdx.x < K_PER_BUCK) {
|
||||
outputKeys[threadIdx.x] += temp;
|
||||
}
|
||||
__syncthreads();
|
||||
}
|
||||
TIME(7)
|
||||
OUTPUT_TIME()
|
||||
}
|
||||
|
||||
void sort(volatile int *keys, int *keyBuffer1,
|
||||
int *keyBuffer2, int *sendCount,
|
||||
int *recvCount, int *sendOffset,
|
||||
int *recvOffset, int *outputKeys,
|
||||
size_t size, int max_iters) {
|
||||
int nProcs, my_pe;
|
||||
MPI_Comm_size(MPI_COMM_WORLD, &nProcs);
|
||||
MPI_Comm_rank(MPI_COMM_WORLD, &my_pe);
|
||||
|
||||
hipStream_t stream;
|
||||
HIPCHECK(hipStreamCreate(&stream));
|
||||
|
||||
for(int iter = 0; iter < max_iters; ++iter) {
|
||||
//fprintf(stderr, "%d: %d %d %p %p\n", my_pe, iter, max_iters, sendCount, recvCount);
|
||||
sort1<<<1, WG_SIZE, 0, stream>>>(keys, keyBuffer1,
|
||||
keyBuffer2, sendCount, recvCount, sendOffset,
|
||||
recvOffset, outputKeys, size, nProcs, my_pe);
|
||||
HIPCHECK(hipStreamSynchronize(stream));
|
||||
MPI_Alltoall(sendCount, 1, MPI_INT, recvCount, 1,
|
||||
MPI_INT, MPI_COMM_WORLD);
|
||||
MPI_Alltoall(sendOffset, 1, MPI_INT, recvOffset, 1,
|
||||
MPI_INT, MPI_COMM_WORLD);
|
||||
int total_size = 0;
|
||||
MPI_Request *req = new MPI_Request[2 * nProcs];
|
||||
const int TAG = 10000;
|
||||
for(int i = 0; i < nProcs; ++i) {
|
||||
MPI_Isend(&keyBuffer1[sendOffset[i]], sendCount[i],
|
||||
MPI_INT, i, TAG, MPI_COMM_WORLD, &req[2 * i]);
|
||||
MPI_Irecv(&keyBuffer2[total_size], recvCount[i],
|
||||
MPI_INT, i, TAG, MPI_COMM_WORLD, &req[2 * i + 1]);
|
||||
total_size += recvCount[i];
|
||||
}
|
||||
MPI_Waitall(2 * nProcs, req, MPI_STATUS_IGNORE);
|
||||
sort2<<<1, WG_SIZE, 0, stream>>>(keys, keyBuffer1,
|
||||
keyBuffer2, sendCount, recvCount, sendOffset,
|
||||
recvOffset, outputKeys, size, nProcs, my_pe);
|
||||
}
|
||||
}
|
||||
|
||||
bool verify(int *outputKeys, int *keyBuffer2, size_t size)
|
||||
{
|
||||
int num_pes, my_pe;
|
||||
MPI_Comm_size(MPI_COMM_WORLD, &num_pes);
|
||||
MPI_Comm_rank(MPI_COMM_WORLD, &my_pe);
|
||||
|
||||
MPI_Status status;
|
||||
MPI_Request request;
|
||||
|
||||
int min_key_val = my_pe * (MAX_KEY / num_pes);
|
||||
int max_key_val = (my_pe + 1) * (MAX_KEY / num_pes) - 1;
|
||||
|
||||
int *key_array = new int[size];
|
||||
// Perform final untimed sort on keys
|
||||
for(int i = 0; i < size; ++i)
|
||||
if(outputKeys[keyBuffer2[i] - min_key_val] > 0)
|
||||
key_array[--outputKeys[keyBuffer2[i] - min_key_val]] = keyBuffer2[i];
|
||||
else {
|
||||
fprintf(stderr, "%d: Found wrong key %d at %d with %d\n", my_pe, keyBuffer2[i], i, outputKeys[keyBuffer2[i]]);
|
||||
return false;
|
||||
}
|
||||
|
||||
if(size < 1)
|
||||
size = 1;
|
||||
|
||||
int k;
|
||||
const int MPI_TAG = 1000;
|
||||
// Check if largest key is smaller than next processor's
|
||||
if(my_pe > 0)
|
||||
MPI_Irecv(&k, 1, MPI_INT, my_pe - 1, MPI_TAG, MPI_COMM_WORLD,
|
||||
&request);
|
||||
if(my_pe < num_pes - 1)
|
||||
MPI_Send(&key_array[size - 1], 1, MPI_INT, my_pe + 1, MPI_TAG,
|
||||
MPI_COMM_WORLD );
|
||||
if(my_pe > 0)
|
||||
MPI_Wait(&request, &status);
|
||||
|
||||
// Check if it is smaller
|
||||
int j = 0;
|
||||
if( my_pe > 0 && size > 1 )
|
||||
if( k > key_array[0] )
|
||||
j++;
|
||||
|
||||
// Check if keys correctly sorted
|
||||
for(int i = 1; i < size; i++)
|
||||
if(key_array[i - 1] > key_array[i])
|
||||
j++;
|
||||
|
||||
delete[] key_array;
|
||||
|
||||
if(j != 0) {
|
||||
fprintf(stderr, "Processor %d: Full_verify: number of keys out of sort: %d\n",
|
||||
my_pe, j );
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
void initGPU()
|
||||
{
|
||||
// Calculation for local rank, taken from rccl-tests
|
||||
int localRank = 0;
|
||||
int nProcs, proc;
|
||||
MPI_Comm_size(MPI_COMM_WORLD, &nProcs);
|
||||
MPI_Comm_rank(MPI_COMM_WORLD, &proc);
|
||||
char hostname[1024];
|
||||
gethostname(hostname, 1024);
|
||||
for (int i=0; i< 1024; i++) {
|
||||
if (hostname[i] == '.') {
|
||||
hostname[i] = '\0';
|
||||
break;
|
||||
}
|
||||
}
|
||||
uint64_t hostHashs[nProcs];
|
||||
hostHashs[proc] = getHostHash(hostname);
|
||||
MPI_Allgather(MPI_IN_PLACE, 0, MPI_DATATYPE_NULL, hostHashs, sizeof(uint64_t), MPI_BYTE, MPI_COMM_WORLD);
|
||||
for (int p=0; p<nProcs; p++) {
|
||||
if (p == proc) break;
|
||||
if (hostHashs[p] == hostHashs[proc]) localRank++;
|
||||
}
|
||||
|
||||
/***
|
||||
* Select a GPU
|
||||
*/
|
||||
int ndevices, my_device=0;
|
||||
hipGetDeviceCount (&ndevices);
|
||||
my_device = localRank % ndevices;
|
||||
hipSetDevice(my_device);
|
||||
|
||||
printf("Rank %d: Device %d, Host %s\n", proc, my_device, hostname);
|
||||
fflush(stdout);
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
}
|
||||
|
||||
void *roc_shmem_malloc(size_t size)
|
||||
{
|
||||
void *v;
|
||||
hipMalloc((void **)&v, size);
|
||||
return v;
|
||||
}
|
||||
|
||||
int roc_shmem_free(void *v)
|
||||
{
|
||||
return hipFree(v);
|
||||
}
|
||||
|
||||
int main(int argc, char *argv[])
|
||||
{
|
||||
if(argc < 1) {
|
||||
printf("Format: %s [iterations]\n", argv[0]);
|
||||
return -1;
|
||||
}
|
||||
|
||||
// Init stuff
|
||||
MPI_Init(&argc, &argv);
|
||||
initGPU();
|
||||
|
||||
int iterations = 1000;
|
||||
if(argc > 1)
|
||||
iterations = atoi(argv[1]);
|
||||
|
||||
int num_pes, my_pe;
|
||||
MPI_Comm_size(MPI_COMM_WORLD, &num_pes);
|
||||
MPI_Comm_rank(MPI_COMM_WORLD, &my_pe);
|
||||
|
||||
// Configure input and outputs
|
||||
size_t size = 1024; //atoi(argv[1]);
|
||||
int *keys, *outputKeys;
|
||||
hipMalloc((void**)&keys, sizeof(int) * size);
|
||||
hipMalloc((void**)&outputKeys, sizeof(int) * WG_SIZE);
|
||||
|
||||
/* Generate random number sequence and subsequent keys on all procs */
|
||||
create_seq( find_my_seed( my_pe,
|
||||
num_pes,
|
||||
4*(long)size*num_pes,
|
||||
314159265.00, /* Random number gen seed */
|
||||
1220703125.00 ), /* Random number gen mult */
|
||||
1220703125.00, keys, size ); /* Random number gen mult */
|
||||
|
||||
|
||||
// Init buffers
|
||||
int *keyBuffer1, *keyBuffer2;
|
||||
keyBuffer1 = (int*)roc_shmem_malloc(sizeof(int) * size);
|
||||
keyBuffer2 = (int*)roc_shmem_malloc(sizeof(int) * size * 4);
|
||||
|
||||
int *sendCount = 0, *recvCount = 0, *sendOffset = 0, *recvOffset = 0;
|
||||
sendCount = (int*)roc_shmem_malloc(sizeof(int) * MAX_PES);
|
||||
recvCount = (int*)roc_shmem_malloc(sizeof(int) * MAX_PES);
|
||||
sendOffset = (int*)roc_shmem_malloc(sizeof(int) * MAX_PES);
|
||||
recvOffset = (int*)roc_shmem_malloc(sizeof(int) * MAX_PES);
|
||||
|
||||
printf("Begin untimed run\n");
|
||||
// Untimed run
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
sort((int*)keys, keyBuffer1, keyBuffer2,
|
||||
sendCount, recvCount, sendOffset, recvOffset,
|
||||
outputKeys, size, 1);
|
||||
hipDeviceSynchronize();
|
||||
|
||||
printf("Verify untimed run\n");
|
||||
// Verify correctness
|
||||
if(!verify(outputKeys, keyBuffer2, outputKeys[MAX_KEY / num_pes - 1])) {
|
||||
fprintf(stderr, "Wrong output\n");
|
||||
return -1;
|
||||
}
|
||||
|
||||
printf("Begin timed run\n");
|
||||
// Timed run
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
auto time_start = TIME_NOW;
|
||||
sort((int*)keys, keyBuffer1, keyBuffer2,
|
||||
sendCount, recvCount, sendOffset, recvOffset,
|
||||
outputKeys, size, iterations);
|
||||
hipDeviceSynchronize();
|
||||
double tot_time = (double)TIME_DIFF(TIME_NOW, time_start);
|
||||
|
||||
double all_time = 0;
|
||||
MPI_Allreduce(&tot_time, &all_time, 1,
|
||||
MPI_DOUBLE, MPI_SUM, MPI_COMM_WORLD);
|
||||
|
||||
if(my_pe == 0) {
|
||||
printf("Avg time:\t%.3f\tus\n", all_time / (double)(1000.0 * iterations * num_pes));
|
||||
}
|
||||
|
||||
// Verify correctness
|
||||
if(!verify(outputKeys, keyBuffer2, outputKeys[MAX_KEY / num_pes - 1])) {
|
||||
fprintf(stderr, "Wrong output\n");
|
||||
return -1;
|
||||
}
|
||||
|
||||
fprintf(stderr, "Done verify for %d\n", my_pe);
|
||||
|
||||
// Clean up
|
||||
hipFree(keys);
|
||||
hipFree(outputKeys);
|
||||
roc_shmem_free(keyBuffer1);
|
||||
roc_shmem_free(keyBuffer2);
|
||||
roc_shmem_free(sendCount);
|
||||
roc_shmem_free(recvCount);
|
||||
roc_shmem_free(sendOffset);
|
||||
roc_shmem_free(recvOffset);
|
||||
MPI_Finalize();
|
||||
return 0;
|
||||
}
|
||||
@@ -0,0 +1,394 @@
|
||||
#include "rccl.h"
|
||||
#include "common.h"
|
||||
#include "sort.h"
|
||||
|
||||
//#define TIME_PERF
|
||||
#ifdef TIME_PERF
|
||||
#define TIMERS 10
|
||||
__device__ uint64_t timers[TIMERS] = {0};
|
||||
__device__ uint64_t time_start;
|
||||
#define TIMERS_START() \
|
||||
if(threadIdx.x == 0) {\
|
||||
time_start = roc_shmem_timer();\
|
||||
}
|
||||
|
||||
#define TIME(TIMER_NUM) \
|
||||
if(threadIdx.x == 0) {\
|
||||
timers[TIMER_NUM] = roc_shmem_timer() - time_start;\
|
||||
time_start = roc_shmem_timer();\
|
||||
}
|
||||
|
||||
#define OUTPUT_TIME() \
|
||||
if(threadIdx.x == 0 && my_pe == 0) { \
|
||||
uint64_t sum = 0; \
|
||||
for(int i = 0; i < TIMERS; ++i) { \
|
||||
sum += timers[i]; \
|
||||
} \
|
||||
for(int i = 0; i < TIMERS; ++i) { \
|
||||
printf("%d: %f\n", i, (double)timers[i] / (double)sum); \
|
||||
} \
|
||||
}
|
||||
#else
|
||||
#define TIMERS_START()
|
||||
#define TIME(x)
|
||||
#define OUTPUT_TIME()
|
||||
#endif
|
||||
|
||||
__global__ void sort1(volatile int *keys, int *keyBuffer1,
|
||||
int *keyBuffer2, int *sendCount,
|
||||
int *recvCount, int *sendOffset,
|
||||
int *recvOffset, int *outputKeys,
|
||||
size_t size, int n_pes, int my_pe) {
|
||||
__shared__ int bucketCounter[MAX_PES];
|
||||
__shared__ int bucketPtr[MAX_PES];
|
||||
__shared__ int total_size;
|
||||
|
||||
int buckets = n_pes;
|
||||
|
||||
int tid = threadIdx.x; // + blockDim.x * blockIdx.x;
|
||||
const int K_PER_BUCK = (MAX_KEY / buckets);
|
||||
|
||||
// Reset
|
||||
for(int i = threadIdx.x; i < buckets; i += blockDim.x) {
|
||||
bucketCounter[i] = 0;
|
||||
bucketPtr[i] = 0;
|
||||
}
|
||||
__syncthreads();
|
||||
TIMERS_START()
|
||||
// Count size of each bucket
|
||||
for(int i = tid; i < size; i += blockDim.x) {
|
||||
atomicAdd(&bucketCounter[keys[i] / K_PER_BUCK], 1);
|
||||
}
|
||||
__syncthreads();
|
||||
TIME(0)
|
||||
// Update in global memory
|
||||
for(int i = tid; i < buckets; i += blockDim.x) {
|
||||
sendCount[i] = bucketPtr[i] = bucketCounter[i];
|
||||
}
|
||||
__syncthreads();
|
||||
TIME(1)
|
||||
// Perform local scan to get ptrs set
|
||||
for(int shift = 1; shift < buckets; shift *= 2) {
|
||||
int temp = 0;
|
||||
if(threadIdx.x >= shift && threadIdx.x < buckets) {
|
||||
temp = bucketPtr[threadIdx.x - shift];
|
||||
}
|
||||
__syncthreads();
|
||||
if(threadIdx.x < buckets) {
|
||||
bucketPtr[threadIdx.x] += temp;
|
||||
}
|
||||
__syncthreads();
|
||||
}
|
||||
__syncthreads();
|
||||
TIME(2)
|
||||
// Find offsets of where we're sending
|
||||
for(int i = threadIdx.x; i < buckets; i += blockDim.x) {
|
||||
sendOffset[i] = bucketPtr[i] - sendCount[i];
|
||||
}
|
||||
// Sort keys into buckets
|
||||
for(int i = threadIdx.x; i < size; i += blockDim.x) {
|
||||
int loc = atomicAdd(&bucketPtr[keys[i] / K_PER_BUCK], -1) - 1;
|
||||
keyBuffer1[loc] = keys[i];
|
||||
}
|
||||
TIME(3)
|
||||
OUTPUT_TIME()
|
||||
}
|
||||
|
||||
__global__ void sort2(volatile int *keys, int *keyBuffer1,
|
||||
int *keyBuffer2, int *sendCount,
|
||||
int *recvCount, int *sendOffset,
|
||||
int *recvOffset, int *outputKeys,
|
||||
size_t size, int n_pes, int my_pe) {
|
||||
__shared__ int total_size;
|
||||
|
||||
int buckets = n_pes;
|
||||
|
||||
int tid = threadIdx.x; // + blockDim.x * blockIdx.x;
|
||||
const int K_PER_BUCK = (MAX_KEY / buckets);
|
||||
|
||||
for(int i = threadIdx.x; i < K_PER_BUCK; i += blockDim.x)
|
||||
outputKeys[i] = 0;
|
||||
__syncthreads();
|
||||
TIME(5)
|
||||
int min_key_val = my_pe * K_PER_BUCK;
|
||||
int max_key_val = (my_pe + 1) * K_PER_BUCK - 1;
|
||||
|
||||
int *key_buff_ptr = outputKeys - min_key_val;
|
||||
for(int i = threadIdx.x; i < total_size; i += blockDim.x) {
|
||||
atomicAdd(&key_buff_ptr[keyBuffer2[i]], 1);
|
||||
}
|
||||
__syncthreads();
|
||||
TIME(6)
|
||||
// Perform local scan on keys
|
||||
for(int shift = 1; shift < K_PER_BUCK; shift *= 2) {
|
||||
int temp = 0;
|
||||
if(threadIdx.x >= shift && threadIdx.x < K_PER_BUCK) {
|
||||
temp = outputKeys[threadIdx.x - shift];
|
||||
}
|
||||
__syncthreads();
|
||||
if(threadIdx.x < K_PER_BUCK) {
|
||||
outputKeys[threadIdx.x] += temp;
|
||||
}
|
||||
__syncthreads();
|
||||
}
|
||||
TIME(7)
|
||||
OUTPUT_TIME()
|
||||
}
|
||||
|
||||
void sort(volatile int *keys, int *keyBuffer1,
|
||||
int *keyBuffer2, int *sendCount,
|
||||
int *recvCount, int *sendOffset,
|
||||
int *recvOffset, int *outputKeys,
|
||||
size_t size, int max_iters, ncclComm_t comm) {
|
||||
int nProcs, my_pe;
|
||||
MPI_Comm_size(MPI_COMM_WORLD, &nProcs);
|
||||
MPI_Comm_rank(MPI_COMM_WORLD, &my_pe);
|
||||
|
||||
hipStream_t stream;
|
||||
HIPCHECK(hipStreamCreate(&stream));
|
||||
|
||||
for(int iter = 0; iter < max_iters; ++iter) {
|
||||
//fprintf(stderr, "%d: %d %d %p %p\n", my_pe, iter, max_iters, sendCount, recvCount);
|
||||
sort1<<<1, WG_SIZE, 0, stream>>>(keys, keyBuffer1,
|
||||
keyBuffer2, sendCount, recvCount, sendOffset,
|
||||
recvOffset, outputKeys, size, nProcs, my_pe);
|
||||
NCCLCHECK(ncclAllToAll(sendCount, recvCount, 1,
|
||||
ncclInt, comm, stream));
|
||||
NCCLCHECK(ncclAllToAll(sendOffset, recvOffset, 1,
|
||||
ncclInt, comm, stream));
|
||||
HIPCHECK(hipStreamSynchronize(stream));
|
||||
NCCLCHECK(ncclGroupStart());
|
||||
int total_size = 0;
|
||||
for(int i = 0; i < nProcs; ++i) {
|
||||
ncclSend(&keyBuffer1[sendOffset[i]], sendCount[i],
|
||||
ncclInt, i, comm, stream);
|
||||
ncclRecv(&keyBuffer2[total_size], recvCount[i],
|
||||
ncclInt, i, comm, stream);
|
||||
total_size += recvCount[i];
|
||||
}
|
||||
NCCLCHECK(ncclGroupEnd());
|
||||
HIPCHECK(hipStreamSynchronize(stream));
|
||||
sort2<<<1, WG_SIZE, 0, stream>>>(keys, keyBuffer1,
|
||||
keyBuffer2, sendCount, recvCount, sendOffset,
|
||||
recvOffset, outputKeys, size, nProcs, my_pe);
|
||||
HIPCHECK(hipStreamSynchronize(stream));
|
||||
}
|
||||
}
|
||||
|
||||
bool verify(int *outputKeys, int *keyBuffer2, size_t size)
|
||||
{
|
||||
int num_pes, my_pe;
|
||||
MPI_Comm_size(MPI_COMM_WORLD, &num_pes);
|
||||
MPI_Comm_rank(MPI_COMM_WORLD, &my_pe);
|
||||
|
||||
MPI_Status status;
|
||||
MPI_Request request;
|
||||
|
||||
int min_key_val = my_pe * (MAX_KEY / num_pes);
|
||||
int max_key_val = (my_pe + 1) * (MAX_KEY / num_pes) - 1;
|
||||
|
||||
int *key_array = new int[size];
|
||||
// Perform final untimed sort on keys
|
||||
for(int i = 0; i < size; ++i)
|
||||
if(outputKeys[keyBuffer2[i] - min_key_val] > 0)
|
||||
key_array[--outputKeys[keyBuffer2[i] - min_key_val]] = keyBuffer2[i];
|
||||
else {
|
||||
fprintf(stderr, "%d: Found wrong key %d at %d with %d\n", my_pe, keyBuffer2[i], i, outputKeys[keyBuffer2[i]]);
|
||||
return false;
|
||||
}
|
||||
|
||||
if(size < 1)
|
||||
size = 1;
|
||||
|
||||
int k;
|
||||
const int MPI_TAG = 1000;
|
||||
// Check if largest key is smaller than next processor's
|
||||
if(my_pe > 0)
|
||||
MPI_Irecv(&k, 1, MPI_INT, my_pe - 1, MPI_TAG, MPI_COMM_WORLD,
|
||||
&request);
|
||||
if(my_pe < num_pes - 1)
|
||||
MPI_Send(&key_array[size - 1], 1, MPI_INT, my_pe + 1, MPI_TAG,
|
||||
MPI_COMM_WORLD );
|
||||
if(my_pe > 0)
|
||||
MPI_Wait(&request, &status);
|
||||
|
||||
// Check if it is smaller
|
||||
int j = 0;
|
||||
if( my_pe > 0 && size > 1 )
|
||||
if( k > key_array[0] )
|
||||
j++;
|
||||
|
||||
// Check if keys correctly sorted
|
||||
for(int i = 1; i < size; i++)
|
||||
if(key_array[i - 1] > key_array[i])
|
||||
j++;
|
||||
|
||||
delete[] key_array;
|
||||
|
||||
if(j != 0) {
|
||||
fprintf(stderr, "Processor %d: Full_verify: number of keys out of sort: %d\n",
|
||||
my_pe, j );
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
void initGPU(ncclComm_t &comms)
|
||||
{
|
||||
// Calculation for local rank, taken from rccl-tests
|
||||
int localRank = 0;
|
||||
int nProcs, proc;
|
||||
MPI_Comm_size(MPI_COMM_WORLD, &nProcs);
|
||||
MPI_Comm_rank(MPI_COMM_WORLD, &proc);
|
||||
char hostname[1024];
|
||||
gethostname(hostname, 1024);
|
||||
for (int i=0; i< 1024; i++) {
|
||||
if (hostname[i] == '.') {
|
||||
hostname[i] = '\0';
|
||||
break;
|
||||
}
|
||||
}
|
||||
uint64_t hostHashs[nProcs];
|
||||
hostHashs[proc] = getHostHash(hostname);
|
||||
MPI_Allgather(MPI_IN_PLACE, 0, MPI_DATATYPE_NULL, hostHashs, sizeof(uint64_t), MPI_BYTE, MPI_COMM_WORLD);
|
||||
for (int p=0; p<nProcs; p++) {
|
||||
if (p == proc) break;
|
||||
if (hostHashs[p] == hostHashs[proc]) localRank++;
|
||||
}
|
||||
|
||||
/***
|
||||
* Select a GPU
|
||||
*/
|
||||
int ndevices, my_device=0;
|
||||
hipGetDeviceCount (&ndevices);
|
||||
my_device = localRank % ndevices;
|
||||
hipSetDevice(my_device);
|
||||
|
||||
ncclUniqueId ncclId;
|
||||
if (proc == 0) {
|
||||
NCCLCHECK(ncclGetUniqueId(&ncclId));
|
||||
}
|
||||
MPI_Bcast(&ncclId, sizeof(ncclId), MPI_BYTE, 0, MPI_COMM_WORLD);
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
|
||||
#ifdef RCCL_MULTIRANKPERGPU
|
||||
NCCLCHECK(ncclCommInitRankMulti(&comms, nProcs, ncclId, proc, proc));
|
||||
#else
|
||||
NCCLCHECK(ncclCommInitRank(&comms, nProcs, ncclId, proc));
|
||||
#endif
|
||||
|
||||
printf("Rank %d: Device %d, Host %s\n", proc, my_device, hostname);
|
||||
fflush(stdout);
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
}
|
||||
|
||||
void *roc_shmem_malloc(size_t size)
|
||||
{
|
||||
void *v;
|
||||
hipMalloc((void **)&v, size);
|
||||
return v;
|
||||
}
|
||||
|
||||
int roc_shmem_free(void *v)
|
||||
{
|
||||
return hipFree(v);
|
||||
}
|
||||
|
||||
int main(int argc, char *argv[])
|
||||
{
|
||||
if(argc < 1) {
|
||||
printf("Format: %s [iterations]\n", argv[0]);
|
||||
return -1;
|
||||
}
|
||||
|
||||
// Init stuff
|
||||
MPI_Init(&argc, &argv);
|
||||
ncclComm_t comms;
|
||||
initGPU(comms);
|
||||
|
||||
int iterations = 1000;
|
||||
if(argc > 1)
|
||||
iterations = atoi(argv[1]);
|
||||
|
||||
int num_pes, my_pe;
|
||||
MPI_Comm_size(MPI_COMM_WORLD, &num_pes);
|
||||
MPI_Comm_rank(MPI_COMM_WORLD, &my_pe);
|
||||
|
||||
// Configure input and outputs
|
||||
size_t size = 1024; //atoi(argv[1]);
|
||||
int *keys, *outputKeys;
|
||||
hipMalloc((void**)&keys, sizeof(int) * size);
|
||||
hipMalloc((void**)&outputKeys, sizeof(int) * WG_SIZE);
|
||||
|
||||
/* Generate random number sequence and subsequent keys on all procs */
|
||||
create_seq( find_my_seed( my_pe,
|
||||
num_pes,
|
||||
4*(long)size*num_pes,
|
||||
314159265.00, /* Random number gen seed */
|
||||
1220703125.00 ), /* Random number gen mult */
|
||||
1220703125.00, keys, size ); /* Random number gen mult */
|
||||
|
||||
|
||||
// Init buffers
|
||||
int *keyBuffer1, *keyBuffer2;
|
||||
keyBuffer1 = (int*)roc_shmem_malloc(sizeof(int) * size);
|
||||
keyBuffer2 = (int*)roc_shmem_malloc(sizeof(int) * size * 4);
|
||||
|
||||
int *sendCount = 0, *recvCount = 0, *sendOffset = 0, *recvOffset = 0;
|
||||
sendCount = (int*)roc_shmem_malloc(sizeof(int) * MAX_PES);
|
||||
recvCount = (int*)roc_shmem_malloc(sizeof(int) * MAX_PES);
|
||||
sendOffset = (int*)roc_shmem_malloc(sizeof(int) * MAX_PES);
|
||||
recvOffset = (int*)roc_shmem_malloc(sizeof(int) * MAX_PES);
|
||||
|
||||
printf("Begin untimed run\n");
|
||||
// Untimed run
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
sort((int*)keys, keyBuffer1, keyBuffer2,
|
||||
sendCount, recvCount, sendOffset, recvOffset,
|
||||
outputKeys, size, 1, comms);
|
||||
hipDeviceSynchronize();
|
||||
|
||||
printf("Verify untimed run\n");
|
||||
// Verify correctness
|
||||
if(!verify(outputKeys, keyBuffer2, outputKeys[MAX_KEY / num_pes - 1])) {
|
||||
fprintf(stderr, "Wrong output\n");
|
||||
return -1;
|
||||
}
|
||||
|
||||
printf("Begin timed run\n");
|
||||
// Timed run
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
auto time_start = TIME_NOW;
|
||||
sort((int*)keys, keyBuffer1, keyBuffer2,
|
||||
sendCount, recvCount, sendOffset, recvOffset,
|
||||
outputKeys, size, iterations, comms);
|
||||
hipDeviceSynchronize();
|
||||
double tot_time = (double)TIME_DIFF(TIME_NOW, time_start);
|
||||
|
||||
double all_time = 0;
|
||||
MPI_Allreduce(&tot_time, &all_time, 1,
|
||||
MPI_DOUBLE, MPI_SUM, MPI_COMM_WORLD);
|
||||
|
||||
if(my_pe == 0) {
|
||||
printf("Avg time:\t%.3f\tus\n", all_time / (double)(1000.0 * iterations * num_pes));
|
||||
}
|
||||
|
||||
// Verify correctness
|
||||
if(!verify(outputKeys, keyBuffer2, outputKeys[MAX_KEY / num_pes - 1])) {
|
||||
fprintf(stderr, "Wrong output\n");
|
||||
return -1;
|
||||
}
|
||||
|
||||
// Clean up
|
||||
hipFree(keys);
|
||||
hipFree(outputKeys);
|
||||
roc_shmem_free(keyBuffer1);
|
||||
roc_shmem_free(keyBuffer2);
|
||||
roc_shmem_free(sendCount);
|
||||
roc_shmem_free(recvCount);
|
||||
roc_shmem_free(sendOffset);
|
||||
roc_shmem_free(recvOffset);
|
||||
ncclCommDestroy(comms);
|
||||
MPI_Finalize();
|
||||
return 0;
|
||||
}
|
||||
@@ -0,0 +1,358 @@
|
||||
#include <iostream>
|
||||
#include <stdio.h>
|
||||
#include <mpi.h>
|
||||
#include <roc_shmem/roc_shmem.hpp>
|
||||
#include <unistd.h>
|
||||
using namespace std;
|
||||
using namespace rocshmem;
|
||||
|
||||
#include "common.h"
|
||||
#include "sort.h"
|
||||
|
||||
//#define TIME_PERF
|
||||
#ifdef TIME_PERF
|
||||
#define TIMERS 10
|
||||
__device__ uint64_t timers[TIMERS] = {0};
|
||||
__device__ uint64_t time_start;
|
||||
#define TIMERS_START() \
|
||||
if(threadIdx.x == 0) {\
|
||||
time_start = roc_shmem_timer();\
|
||||
}
|
||||
|
||||
#define TIME(TIMER_NUM) \
|
||||
if(threadIdx.x == 0) {\
|
||||
timers[TIMER_NUM] = roc_shmem_timer() - time_start;\
|
||||
time_start = roc_shmem_timer();\
|
||||
}
|
||||
|
||||
#define OUTPUT_TIME() \
|
||||
if(threadIdx.x == 0 && my_pe == 0) { \
|
||||
uint64_t sum = 0; \
|
||||
for(int i = 0; i < TIMERS; ++i) { \
|
||||
sum += timers[i]; \
|
||||
} \
|
||||
for(int i = 0; i < TIMERS; ++i) { \
|
||||
printf("%d: %f\n", i, (double)timers[i] / (double)sum); \
|
||||
} \
|
||||
}
|
||||
#else
|
||||
#define TIMERS_START()
|
||||
#define TIME(x)
|
||||
#define OUTPUT_TIME()
|
||||
#endif
|
||||
|
||||
__device__ __inline__ void alltoall(roc_shmem_ctx_t &ctx,
|
||||
roc_shmem_team_t team,
|
||||
int *dst, int *src) {
|
||||
// Perform alltoall
|
||||
roc_shmem_ctx_int_wg_alltoall(ctx,
|
||||
team,
|
||||
dst, // T* dest
|
||||
src, // const T* source
|
||||
1); // int nelement
|
||||
}
|
||||
|
||||
__global__ void sort(volatile int *keys, int *keyBuffer1,
|
||||
int *keyBuffer2, int *sendCount,
|
||||
int *recvCount, int *sendOffset,
|
||||
int *recvOffset, int *outputKeys,
|
||||
size_t size, roc_shmem_team_t team,
|
||||
int max_iters) {
|
||||
__shared__ roc_shmem_ctx_t ctx;
|
||||
__shared__ int bucketCounter[MAX_PES];
|
||||
__shared__ int bucketPtr[MAX_PES];
|
||||
__shared__ int total_size;
|
||||
|
||||
roc_shmem_wg_init();
|
||||
roc_shmem_wg_ctx_create(ROC_SHMEM_CTX_WG_PRIVATE, &ctx);
|
||||
|
||||
int n_pes = roc_shmem_ctx_n_pes(ctx);
|
||||
int my_pe = roc_shmem_my_pe();
|
||||
int buckets = n_pes;
|
||||
|
||||
int tid = threadIdx.x; // + blockDim.x * blockIdx.x;
|
||||
const int K_PER_BUCK = (MAX_KEY / buckets);
|
||||
|
||||
for(int iter = 0; iter < max_iters; ++iter) {
|
||||
// Reset
|
||||
for(int i = threadIdx.x; i < buckets; i += blockDim.x) {
|
||||
bucketCounter[i] = 0;
|
||||
bucketPtr[i] = 0;
|
||||
}
|
||||
__syncthreads();
|
||||
TIMERS_START()
|
||||
// Count size of each bucket
|
||||
for(int i = tid; i < size; i += blockDim.x) {
|
||||
atomicAdd(&bucketCounter[keys[i] / K_PER_BUCK], 1);
|
||||
}
|
||||
__syncthreads();
|
||||
TIME(0)
|
||||
// Update in global memory
|
||||
for(int i = tid; i < buckets; i += blockDim.x) {
|
||||
sendCount[i] = bucketPtr[i] = bucketCounter[i];
|
||||
}
|
||||
__syncthreads();
|
||||
TIME(1)
|
||||
// Perform local scan to get ptrs set
|
||||
for(int shift = 1; shift < buckets; shift *= 2) {
|
||||
int temp = 0;
|
||||
if(threadIdx.x >= shift && threadIdx.x < buckets) {
|
||||
temp = bucketPtr[threadIdx.x - shift];
|
||||
}
|
||||
__syncthreads();
|
||||
if(threadIdx.x < buckets) {
|
||||
bucketPtr[threadIdx.x] += temp;
|
||||
}
|
||||
__syncthreads();
|
||||
}
|
||||
__syncthreads();
|
||||
TIME(2)
|
||||
// Find offsets of where we're sending
|
||||
for(int i = threadIdx.x; i < buckets; i += blockDim.x) {
|
||||
sendOffset[i] = bucketPtr[i] - sendCount[i];
|
||||
}
|
||||
// Sort keys into buckets
|
||||
for(int i = threadIdx.x; i < size; i += blockDim.x) {
|
||||
int loc = atomicAdd(&bucketPtr[keys[i] / K_PER_BUCK], -1) - 1;
|
||||
keyBuffer1[loc] = keys[i];
|
||||
}
|
||||
roc_shmem_ctx_threadfence_system(ctx);
|
||||
// Force sync to wait for all PEs to update bucket sizes
|
||||
roc_shmem_ctx_wg_team_sync(ctx, team);
|
||||
TIME(3)
|
||||
// Let all PEs know how many keys you wish to send
|
||||
alltoall(ctx, team, recvCount, sendCount);
|
||||
// Let all PEs know where the offsets are of the keys
|
||||
alltoall(ctx, team, recvOffset, sendOffset);
|
||||
__syncthreads();
|
||||
TIME(4)
|
||||
if(threadIdx.x == 0) {
|
||||
total_size = 0;
|
||||
for(int i = 0; i < buckets; ++i) {
|
||||
roc_shmem_int_get_nbi(&keyBuffer2[total_size],
|
||||
&keyBuffer1[recvOffset[i]], recvCount[i], i);
|
||||
total_size += recvCount[i];
|
||||
}
|
||||
roc_shmem_quiet();
|
||||
}
|
||||
for(int i = threadIdx.x; i < K_PER_BUCK; i += blockDim.x)
|
||||
outputKeys[i] = 0;
|
||||
__syncthreads();
|
||||
TIME(5)
|
||||
int min_key_val = my_pe * K_PER_BUCK;
|
||||
int max_key_val = (my_pe + 1) * K_PER_BUCK - 1;
|
||||
|
||||
int *key_buff_ptr = outputKeys - min_key_val;
|
||||
for(int i = threadIdx.x; i < total_size; i += blockDim.x) {
|
||||
atomicAdd(&key_buff_ptr[keyBuffer2[i]], 1);
|
||||
}
|
||||
__syncthreads();
|
||||
TIME(6)
|
||||
// Perform local scan on keys
|
||||
for(int shift = 1; shift < K_PER_BUCK; shift *= 2) {
|
||||
int temp = 0;
|
||||
if(threadIdx.x >= shift && threadIdx.x < K_PER_BUCK) {
|
||||
temp = outputKeys[threadIdx.x - shift];
|
||||
}
|
||||
__syncthreads();
|
||||
if(threadIdx.x < K_PER_BUCK) {
|
||||
outputKeys[threadIdx.x] += temp;
|
||||
}
|
||||
__syncthreads();
|
||||
}
|
||||
TIME(7)
|
||||
}
|
||||
OUTPUT_TIME()
|
||||
roc_shmem_wg_ctx_destroy(ctx);
|
||||
roc_shmem_wg_finalize();
|
||||
}
|
||||
|
||||
bool verify(int *outputKeys, int *keyBuffer2, size_t size)
|
||||
{
|
||||
int num_pes = roc_shmem_n_pes();
|
||||
int my_pe = roc_shmem_my_pe();
|
||||
|
||||
MPI_Status status;
|
||||
MPI_Request request;
|
||||
|
||||
int min_key_val = my_pe * (MAX_KEY / num_pes);
|
||||
int max_key_val = (my_pe + 1) * (MAX_KEY / num_pes) - 1;
|
||||
|
||||
int *key_array = new int[size];
|
||||
// Perform final untimed sort on keys
|
||||
for(int i = 0; i < size; ++i)
|
||||
if(outputKeys[keyBuffer2[i] - min_key_val] > 0)
|
||||
key_array[--outputKeys[keyBuffer2[i] - min_key_val]] = keyBuffer2[i];
|
||||
else {
|
||||
fprintf(stderr, "%d: Found wrong key %d at %d with %d\n", my_pe, keyBuffer2[i], i, outputKeys[keyBuffer2[i]]);
|
||||
return false;
|
||||
}
|
||||
|
||||
if(size < 1)
|
||||
size = 1;
|
||||
|
||||
int k;
|
||||
const int MPI_TAG = 1000;
|
||||
// Check if largest key is smaller than next processor's
|
||||
if(my_pe > 0)
|
||||
MPI_Irecv(&k, 1, MPI_INT, my_pe - 1, MPI_TAG, MPI_COMM_WORLD,
|
||||
&request);
|
||||
if(my_pe < num_pes - 1)
|
||||
MPI_Send(&key_array[size - 1], 1, MPI_INT, my_pe + 1, MPI_TAG,
|
||||
MPI_COMM_WORLD );
|
||||
if(my_pe > 0)
|
||||
MPI_Wait(&request, &status);
|
||||
|
||||
// Check if it is smaller
|
||||
int j = 0;
|
||||
if( my_pe > 0 && size > 1 )
|
||||
if( k > key_array[0] )
|
||||
j++;
|
||||
|
||||
// Check if keys correctly sorted
|
||||
for(int i = 1; i < size; i++)
|
||||
if(key_array[i - 1] > key_array[i])
|
||||
j++;
|
||||
|
||||
delete[] key_array;
|
||||
|
||||
if(j != 0) {
|
||||
fprintf(stderr, "Processor %d: Full_verify: number of keys out of sort: %d\n",
|
||||
my_pe, j );
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
void initGPU()
|
||||
{
|
||||
// Calculation for local rank, taken from rccl-tests
|
||||
int localRank = 0;
|
||||
int proc = roc_shmem_my_pe();
|
||||
int nProcs = roc_shmem_n_pes();
|
||||
char hostname[1024];
|
||||
gethostname(hostname, 1024);
|
||||
for (int i=0; i< 1024; i++) {
|
||||
if (hostname[i] == '.') {
|
||||
hostname[i] = '\0';
|
||||
break;
|
||||
}
|
||||
}
|
||||
uint64_t hostHashs[nProcs];
|
||||
hostHashs[proc] = getHostHash(hostname);
|
||||
MPI_Allgather(MPI_IN_PLACE, 0, MPI_DATATYPE_NULL, hostHashs, sizeof(uint64_t), MPI_BYTE, MPI_COMM_WORLD);
|
||||
for (int p=0; p<nProcs; p++) {
|
||||
if (p == proc) break;
|
||||
if (hostHashs[p] == hostHashs[proc]) localRank++;
|
||||
}
|
||||
|
||||
/***
|
||||
* Select a GPU
|
||||
*/
|
||||
int ndevices, my_device=0;
|
||||
hipGetDeviceCount (&ndevices);
|
||||
my_device = localRank % ndevices;
|
||||
hipSetDevice(my_device);
|
||||
|
||||
printf("Rank %d: Device %d, Host %s\n", proc, my_device, hostname);
|
||||
fflush(stdout);
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
}
|
||||
|
||||
int main(int argc, char *argv[])
|
||||
{
|
||||
// Init roc_shmem stuff
|
||||
initGPU();
|
||||
roc_shmem_init(NUM_WGS);
|
||||
int n_pes = roc_shmem_team_n_pes(ROC_SHMEM_TEAM_WORLD);
|
||||
roc_shmem_team_t team_world_dup = ROC_SHMEM_TEAM_INVALID;
|
||||
roc_shmem_team_split_strided(ROC_SHMEM_TEAM_WORLD,
|
||||
0,
|
||||
1,
|
||||
n_pes,
|
||||
nullptr,
|
||||
0,
|
||||
&team_world_dup);
|
||||
|
||||
int iterations = 1000;
|
||||
if(argc > 1)
|
||||
iterations = atoi(argv[1]);
|
||||
|
||||
int num_pes = roc_shmem_n_pes();
|
||||
int my_pe = roc_shmem_my_pe();
|
||||
|
||||
// Configure input and outputs
|
||||
size_t size = 1024; //atoi(argv[2]);
|
||||
int *keys, *outputKeys;
|
||||
hipMalloc((void**)&keys, sizeof(int) * size);
|
||||
hipMalloc((void**)&outputKeys, sizeof(int) * WG_SIZE);
|
||||
|
||||
/* Generate random number sequence and subsequent keys on all procs */
|
||||
create_seq( find_my_seed( my_pe,
|
||||
num_pes,
|
||||
4*(long)size*num_pes,
|
||||
314159265.00, /* Random number gen seed */
|
||||
1220703125.00 ), /* Random number gen mult */
|
||||
1220703125.00, keys, size ); /* Random number gen mult */
|
||||
|
||||
|
||||
// Init buffers
|
||||
int *keyBuffer1, *keyBuffer2;
|
||||
keyBuffer1 = (int*)roc_shmem_malloc(sizeof(int) * size);
|
||||
keyBuffer2 = (int*)roc_shmem_malloc(sizeof(int) * size * 4);
|
||||
|
||||
int *sendCount, *recvCount, *sendOffset, *recvOffset;
|
||||
sendCount = (int*)roc_shmem_malloc(sizeof(int) * MAX_PES);
|
||||
recvCount = (int*)roc_shmem_malloc(sizeof(int) * MAX_PES);
|
||||
sendOffset = (int*)roc_shmem_malloc(sizeof(int) * MAX_PES);
|
||||
recvOffset = (int*)roc_shmem_malloc(sizeof(int) * MAX_PES);
|
||||
|
||||
// Untimed run
|
||||
roc_shmem_barrier_all();
|
||||
sort<<<1, WG_SIZE>>>((int*)keys, keyBuffer1, keyBuffer2,
|
||||
sendCount, recvCount, sendOffset, recvOffset,
|
||||
outputKeys, size, team_world_dup, 1);
|
||||
hipDeviceSynchronize();
|
||||
|
||||
// Verify correctness
|
||||
if(!verify(outputKeys, keyBuffer2, outputKeys[MAX_KEY / num_pes - 1])) {
|
||||
fprintf(stderr, "Wrong output\n");
|
||||
return -1;
|
||||
}
|
||||
|
||||
// Timed run
|
||||
roc_shmem_barrier_all();
|
||||
auto time_start = TIME_NOW;
|
||||
sort<<<1, WG_SIZE>>>((int*)keys, keyBuffer1, keyBuffer2,
|
||||
sendCount, recvCount, sendOffset, recvOffset,
|
||||
outputKeys, size, team_world_dup, iterations);
|
||||
hipDeviceSynchronize();
|
||||
double tot_time = (double)TIME_DIFF(TIME_NOW, time_start);
|
||||
|
||||
double all_time = 0;
|
||||
MPI_Allreduce(&tot_time, &all_time, 1,
|
||||
MPI_DOUBLE, MPI_SUM, MPI_COMM_WORLD);
|
||||
|
||||
if(my_pe == 0) {
|
||||
printf("Avg time:\t%f\tus\n", all_time /
|
||||
(double)(1000.0 * iterations * num_pes));
|
||||
}
|
||||
|
||||
// Verify correctness
|
||||
if(!verify(outputKeys, keyBuffer2, outputKeys[MAX_KEY / num_pes - 1])) {
|
||||
fprintf(stderr, "Wrong output\n");
|
||||
return -1;
|
||||
}
|
||||
|
||||
// Clean up
|
||||
hipFree(keys);
|
||||
hipFree(outputKeys);
|
||||
roc_shmem_free(keyBuffer1);
|
||||
roc_shmem_free(keyBuffer2);
|
||||
roc_shmem_free(sendCount);
|
||||
roc_shmem_free(recvCount);
|
||||
roc_shmem_free(sendOffset);
|
||||
roc_shmem_free(recvOffset);
|
||||
roc_shmem_finalize();
|
||||
return 0;
|
||||
}
|
||||
Reference in New Issue
Block a user