Submitting jobs to cdash (#124)

* Submitting jobs to cdash * Fail on submit * submit url env * submit url env * try passing submit url as arg * fix submit url * Updated default URL * Add submissions for remaining ubuntu focal workflow jobs * Replace g++ with gcc in dashboard build name * Add --ctest-args to run-ci.sh * Add cdash support for bionic, jammy, and opensuse workflows * Decrease CTEST_CUSTOM_MAXIMUM_PASSED_TEST_OUTPUT_SIZE * OMNITRACE_BUILD_CODECOV option * Support code coverage in CDash script * CI dyninst built with debug info * Update ci-containers - cron schedule moved 4 hours later to UTC+5 * Update implementation of config::configure_signal_handler - using lambdas failed to compile with codecov flags * Add codecov job to ubuntu focal workflow * Fix support for --ctest-args in run-ci script * Fix ubuntu workflows * Fix quotation handling in run-ci script * git safe directory for codecov * New MPI examples * Remove --stop-on-failure * dynamic_library update - find_library_path checks procfs maps - invoke find_library_path with no additional args to resolve to mapped file * RCCLP uses dynamic_library * check if file exists for memory_map_files metadata * Testing updates - include new mpi examples in tests - fix test labels - test critical-trace exe * Update MPI C examples tests (needed arg) * Remove try/catch block from critical-trace * Fix sampling max wait when shutting down * Fix test env for critical-trace * Fix settings for critical-trace - disable time output: data is deterministic - disable PID suffixes: not multiprocess * Update critical-trace ctest * Update critical-trace exe - throw error if input cannot be opened - throw error if input has no data * Update lulesh example with more kokkos tools usage * Fix tasking issue with critical_trace and roctracer - were not setting pools to active - also sync before critical_trace::get_entries * Increase verbosity of critical-trace tests * Update code coverage tests - skip code coverage + preload - code-coverage python example and test * Remove duplication omnitrace.initialize function * Skip python3.6 for ubuntu jammy * Update MPI examples - use MPI_Isend and MPI_Irecv - explicitly use MPI_Bcast * Update Formatting.cmake - include C files in examples * run-ci script does not check return of coverage * mpi-allreduce link to libm * Update ctest args in run-ci script * Update dyninst submodule - safety improvements in BinaryEdit::openResolvedLibraryName * capture cmake error for ctest_coverage [ROCm/rocprofiler-systems commit: 46b6db1a4c]
2022-10-31 15:39:45 -05:00
parent 268f94be4b
commit c87e69e522
34 changed files with 1698 additions and 230 deletions
@@ -16,3 +16,23 @@ if(OMNITRACE_INSTALL_EXAMPLES)
        DESTINATION bin
        COMPONENT omnitrace-examples)
 endif()
+
+set(PYTHON_FILES code-coverage.py)
+
+find_package(Python3 COMPONENTS Interpreter)
+
+if(Python3_FOUND)
+    set(PYTHON_EXECUTABLE "${Python3_EXECUTABLE}")
+
+    foreach(_FILE ${PYTHON_FILES})
+        configure_file(${PROJECT_SOURCE_DIR}/${_FILE} ${PROJECT_BINARY_DIR}/${_FILE}
+                       @ONLY)
+
+        if(OMNITRACE_INSTALL_EXAMPLES)
+            install(
+                PROGRAMS ${PROJECT_BINARY_DIR}/${_FILE}
+                DESTINATION bin
+                COMPONENT omnitrace-examples)
+        endif()
+    endforeach()
+endif()
@@ -0,0 +1,54 @@
+#!@PYTHON_EXECUTABLE@
+
+import omnitrace
+import argparse
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+
+    parser.add_argument(
+        "-i",
+        "--input",
+        type=str,
+        nargs="+",
+        help="Input code coverage",
+        default=None,
+        required=True,
+    )
+    parser.add_argument(
+        "-o",
+        "--output",
+        type=str,
+        help="Output code coverage",
+        default=None,
+        required=True,
+    )
+
+    args = parser.parse_args()
+
+    data = None
+    for itr in args.input:
+        _summary, _details = omnitrace.coverage.load(itr)
+        if data is None:
+            data = _details
+        else:
+            data = omnitrace.coverage.concat(data, _details)
+
+    summary = omnitrace.coverage.get_summary(data)
+    top = omnitrace.coverage.get_top(data)
+    bottom = omnitrace.coverage.get_bottom(data)
+
+    print("Top code coverage:")
+    for itr in top:
+        print(
+            f"    {itr.count} | {itr.function} | {itr.module}:{itr.line} | {itr.source}"
+        )
+
+    print("Bottom code coverage:")
+    for itr in bottom:
+        print(
+            f"    {itr.count} | {itr.function} | {itr.module}:{itr.line} | {itr.source}"
+        )
+
+    print("\nSaving code coverage")
+    omnitrace.coverage.save(summary, data, args.output)
@@ -751,7 +751,7 @@ CalcHourglassControlForElems(Domain& domain, Real_t determ[], Real_t hgcoef)

    int error = 0;
    Kokkos::parallel_reduce(
-        numElem,
+        "CalcHourglassControlForElems", numElem,
        KOKKOS_LAMBDA(const int i, int& err) {
            Real_t x1[8], y1[8], z1[8];

@@ -813,7 +813,7 @@ CalcVolumeForceForElems(Domain& domain)
        // check for negative element volume
        int error = 0;
        Kokkos::parallel_reduce(
-            numElem,
+            "CalcVolumeForceForElems", numElem,
            KOKKOS_LAMBDA(const int k, int& err) {
                if(determ[k] <= Real_t(0.0))
                {
@@ -1221,7 +1221,7 @@ CalcLagrangeElements(Domain& domain)

        int error = 0;
        Kokkos::parallel_reduce(
-            numElem,
+            "CalcLagrangeElements", numElem,
            KOKKOS_LAMBDA(const int k, int& err) {
                Real_t vdov      = domain.dxx(k) + domain.dyy(k) + domain.dzz(k);
                Real_t vdovthird = vdov / Real_t(3.0);
@@ -1607,7 +1607,7 @@ CalcQForElems(Domain& domain)

        Index_t idx = 0;
        Kokkos::parallel_reduce(
-            numElem,
+            "CalcQForElems", numElem,
            KOKKOS_LAMBDA(const Index_t& i, Index_t& count) {
                if(domain.q(i) > domain.qstop())
                {
@@ -1931,7 +1931,7 @@ ApplyMaterialPropertiesForElems(Domain& domain)

        int error = 0;
        Kokkos::parallel_reduce(
-            numElem,
+            "ApplyMaterialPropertiesForElems", numElem,
            KOKKOS_LAMBDA(const int i, int& err) {
                Real_t vc = domain.v(i);
                if(eosvmin != Real_t(0.))
@@ -1949,7 +1949,7 @@ ApplyMaterialPropertiesForElems(Domain& domain)
            },
            error);

-        if(error)
+        if(error != 0)
 #if USE_MPI
            MPI_Abort(MPI_COMM_WORLD, VolumeError);
 #else
@@ -2015,7 +2015,7 @@ CalcCourantConstraintForElems(Domain& domain, Index_t length, Index_t r, Real_t
    MinFinder result;

    Kokkos::parallel_reduce(
-        length,
+        "CalcCourantConstraintForElems", length,
        KOKKOS_LAMBDA(const int i, MinFinder& minf) {
            Index_t indx = domain.regElemlist(r, i);
            Real_t  dtf  = domain.ss(indx) * domain.ss(indx);
@@ -2065,7 +2065,7 @@ CalcHydroConstraintForElems(Domain& domain, Index_t length, Index_t r, Real_t dv
    MinFinder result;

    Kokkos::parallel_reduce(
-        length,
+        "CalcHydroConstraintForElems", length,
        KOKKOS_LAMBDA(const int i, MinFinder& minf) {
            Index_t indx = domain.regElemlist(r, i);

@@ -2167,8 +2167,9 @@ main(int argc, char* argv[])
    myRank = 0;
 #endif

-    Kokkos::initialize();
+    Kokkos::initialize(argc, argv);
    {
+        Kokkos::Tools::pushRegion("initialization");
        opts.its       = 9999999;
        opts.nx        = 30;
        opts.numReg    = 11;
@@ -2223,24 +2224,41 @@ main(int argc, char* argv[])
        MPI_Barrier(MPI_COMM_WORLD);
 #endif

+        Kokkos::Tools::popRegion();
+
 #if USE_MPI
        double start = MPI_Wtime();
 #else
        timeval start;
-        gettimeofday(&start, NULL);
+        gettimeofday(&start, nullptr);
 #endif
+
+        uint32_t _time_incrp = 0;
+        uint32_t _leap_frogp = 0;
+        Kokkos::Tools::createProfileSection("TimeIncr", &_time_incrp);
+        Kokkos::Tools::createProfileSection("LeapFrog", &_leap_frogp);
+
        while((locDom.time() < locDom.stoptime()) && (locDom.cycle() < opts.its))
        {
+            Kokkos::Tools::startSection(_time_incrp);
            TimeIncrement(locDom);
+            Kokkos::Tools::stopSection(_time_incrp);
+
+            Kokkos::Tools::startSection(_leap_frogp);
            LagrangeLeapFrog(locDom);
+            Kokkos::Tools::stopSection(_leap_frogp);

            if((opts.showProg != 0) && (opts.quiet == 0) && (myRank == 0))
            {
                printf("cycle = %d, time = %e, dt=%e\n", locDom.cycle(),
                       double(locDom.time()), double(locDom.deltatime()));
            }
+            Kokkos::Tools::markEvent("completed_timestep");
        }

+        Kokkos::Tools::destroyProfileSection(_time_incrp);
+        Kokkos::Tools::destroyProfileSection(_leap_frogp);
+
        double elapsed_time;
 #if USE_MPI
        elapsed_time = MPI_Wtime() - start;
@@ -2258,6 +2276,7 @@ main(int argc, char* argv[])
        elapsed_timeG = elapsed_time;
 #endif

+        Kokkos::Tools::pushRegion("finalization");
        if(opts.viz)
        {
            DumpToVisit(locDom, opts.numFiles, myRank, numRanks);
@@ -2267,6 +2286,7 @@ main(int argc, char* argv[])
        {
            VerifyAndWriteFinalOutput(elapsed_timeG, locDom, opts.nx, numRanks);
        }
+        Kokkos::Tools::popRegion();

        buffer = Kokkos::View<Real_t*>();
    }
@@ -1,8 +1,6 @@
 cmake_minimum_required(VERSION 3.16 FATAL_ERROR)

-project(omnitrace-mpi-example LANGUAGES CXX)
-
-set(CMAKE_BUILD_TYPE "Release")
+project(omnitrace-mpi-examples LANGUAGES C CXX)

 find_package(MPI)
 if(NOT MPI_FOUND)
@@ -12,15 +10,51 @@ endif()

 find_package(Threads REQUIRED)

-add_executable(mpi-example mpi.cpp)
+set(CMAKE_BUILD_TYPE "RelWithDebInfo")

+add_library(mpi-c-interface-library INTERFACE)
 target_link_libraries(
-    mpi-example PRIVATE MPI::MPI_CXX Threads::Threads
-                        $<TARGET_NAME_IF_EXISTS:omnitrace::omnitrace-compile-options>)
+    mpi-c-interface-library
+    INTERFACE Threads::Threads MPI::MPI_C
+              $<TARGET_NAME_IF_EXISTS:omnitrace::omnitrace-compile-options>)
+target_compile_options(mpi-c-interface-library INTERFACE -Wno-double-promotion)
+
+add_executable(mpi-allgather allgather.c)
+target_link_libraries(mpi-allgather PRIVATE mpi-c-interface-library)
+
+add_executable(mpi-bcast bcast.c)
+target_link_libraries(mpi-bcast PRIVATE mpi-c-interface-library)
+
+add_executable(mpi-all2all all2all.c)
+target_link_libraries(mpi-all2all PRIVATE mpi-c-interface-library)
+
+add_executable(mpi-reduce reduce.c)
+target_link_libraries(mpi-reduce PRIVATE mpi-c-interface-library)
+
+add_executable(mpi-scatter-gather scatter-gather.c)
+target_link_libraries(mpi-scatter-gather PRIVATE mpi-c-interface-library)
+
+add_executable(mpi-send-recv send-recv.c)
+target_link_libraries(mpi-send-recv PRIVATE mpi-c-interface-library)
+
+add_executable(mpi-allreduce allreduce.c)
+target_link_libraries(mpi-allreduce PRIVATE mpi-c-interface-library m)
+
+set(CMAKE_BUILD_TYPE "Release")
+
+add_library(mpi-cxx-interface-library INTERFACE)
+target_link_libraries(
+    mpi-cxx-interface-library
+    INTERFACE Threads::Threads MPI::MPI_CXX
+              $<TARGET_NAME_IF_EXISTS:omnitrace::omnitrace-compile-options>)
+
+add_executable(mpi-example mpi.cpp)
+target_link_libraries(mpi-example PRIVATE mpi-cxx-interface-library)

 if(OMNITRACE_INSTALL_EXAMPLES)
    install(
-        TARGETS mpi-example
+        TARGETS mpi-example mpi-allgather mpi-bcast mpi-all2all mpi-reduce
+                mpi-scatter-gather mpi-send-recv
        DESTINATION bin
        COMPONENT omnitrace-examples)
 endif()
@@ -0,0 +1,244 @@
+// Author: Wes Kendall
+// Copyright 2014 www.mpitutorial.com
+// This code is provided freely with the tutorials on mpitutorial.com. Feel
+// free to modify it for your own use. Any distribution of the code must
+// either provide a link to www.mpitutorial.com or keep this header intact.
+//
+// A program that bins random numbers using MPI_Alltoallv.
+//
+#include <mpi.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <time.h>
+
+// Creates an array of random numbers for binning. Note that the numbers are
+// between [0, 1)
+float*
+create_random_numbers(int numbers_per_proc)
+{
+    float* random_numbers = (float*) malloc(sizeof(float) * numbers_per_proc);
+    int    i;
+    for(i = 0; i < numbers_per_proc; i++)
+    {
+        int r = rand();
+        // Make sure that the random number is never exactly one.
+        if(r == RAND_MAX)
+        {
+            r--;
+        }
+        random_numbers[i] = rand() / (float) (RAND_MAX);
+    }
+    return random_numbers;
+}
+
+// Given a number, determine which process owns it. Since numbers are from [0, 1),
+// simply multiple the number by the size of the MPI world to figure out which
+// process owns it
+int
+which_process_owns_this_number(float rand_num, int world_size)
+{
+    return (int) (rand_num * world_size);
+}
+
+// Gets the starting value for a process's bin
+float
+get_bin_start(int world_rank, int world_size)
+{
+    return (float) world_rank / world_size;
+}
+
+// Gets the ending value for a process's bin
+float
+get_bin_end(int world_rank, int world_size)
+{
+    return get_bin_start(world_rank + 1, world_size);
+}
+
+// This function returns the amount of numbers that will be sent to each
+// process given the array of random numbers.
+int*
+get_send_amounts_per_proc(float* rand_nums, int numbers_per_proc, int world_size)
+{
+    int* send_amounts_per_proc = (int*) malloc(sizeof(int) * world_size);
+    // Initialize the amount of numbers per process to zero
+    memset(send_amounts_per_proc, 0, sizeof(int) * world_size);
+
+    // For each random number, determine which process owns it and increment
+    // the amount of numbers for that process.
+    int i;
+    for(i = 0; i < numbers_per_proc; i++)
+    {
+        int owning_rank = which_process_owns_this_number(rand_nums[i], world_size);
+        send_amounts_per_proc[owning_rank]++;
+    }
+
+    return send_amounts_per_proc;
+}
+
+// Given how many numbers each process is sending to the other processes, find
+// out how many numbers you are receiving from each process. This function
+// returns an array of counts indexed on the rank of the process from which it
+// will receive the numbers.
+int*
+get_recv_amounts_per_proc(int* send_amounts_per_proc, int world_size)
+{
+    int* recv_amounts_per_proc = (int*) malloc(sizeof(int) * world_size);
+
+    // Perform an Alltoall for the send counts. This will send the send counts
+    // from each process and place them in the recv_amounts_per_proc array of
+    // the receiving processes to let them know how many numbers they will
+    // receive when binning occurs.
+    MPI_Alltoall(send_amounts_per_proc, 1, MPI_INT, recv_amounts_per_proc, 1, MPI_INT,
+                 MPI_COMM_WORLD);
+    return recv_amounts_per_proc;
+}
+
+// Given an array (of size "size") of counts, return the prefix sum of the
+// counts.
+int*
+prefix_sum(const int* counts, int size)
+{
+    int* prefix_sum_result = (int*) malloc(sizeof(int) * size);
+    prefix_sum_result[0]   = 0;
+    int i;
+    for(i = 1; i < size; i++)
+    {
+        prefix_sum_result[i] = prefix_sum_result[i - 1] + counts[i - 1];
+    }
+    return prefix_sum_result;
+}
+
+// Returns the sum of an array
+int
+sum(const int* arr, int size)
+{
+    int sum_result = 0;
+    int i;
+    for(i = 0; i < size; i++)
+    {
+        sum_result += arr[i];
+    }
+    return sum_result;
+}
+
+// Used for sorting floating point numbers
+int
+compare_float(const void* a, const void* b)
+{
+    if(*(float*) a < *(float*) b)
+    {
+        return -1;
+    }
+    else if(*(float*) a > *(float*) b)
+    {
+        return 1;
+    }
+    else
+    {
+        return 0;
+    }
+}
+
+// Verifies that the binned numbers belong to the process.
+void
+verify_bin_nums(float* binned_nums, int num_count, int world_rank, int world_size)
+{
+    int   i;
+    float bin_start = get_bin_start(world_rank, world_size);
+    float bin_end   = get_bin_end(world_rank, world_size);
+    for(i = 0; i < num_count; i++)
+    {
+        if(binned_nums[i] >= bin_end || binned_nums[i] < bin_start)
+        {
+            fprintf(
+                stderr,
+                "Error: Binned number %f exceeds bin range [%f - %f) for process %d\n",
+                binned_nums[i], bin_start, bin_end, world_rank);
+        }
+    }
+}
+
+int
+main(int argc, char** argv)
+{
+    if(argc != 2)
+    {
+        fprintf(stderr, "Usage: bin numbers_per_proc\n");
+        exit(1);
+    }
+
+    // Get the amount of random numbers to create per process
+    int numbers_per_proc = atoi(argv[1]);
+
+    MPI_Init(NULL, NULL);
+
+    int world_rank;
+    MPI_Comm_rank(MPI_COMM_WORLD, &world_rank);
+    int world_size;
+    MPI_Comm_size(MPI_COMM_WORLD, &world_size);
+
+    // Seed the random number generator to get different results each time
+    srand(time(NULL) * world_rank);
+
+    // Create the random numbers on this process. Note that all numbers
+    // will be between 0 and 1
+    float* rand_nums = create_random_numbers(numbers_per_proc);
+
+    // Given the array of random numbers, determine how many will be sent
+    // to each process (based on the which process owns the number).
+    // The return value from this function is an array of counts
+    // for each rank in the communicator.
+    // The count represents how many numbers each process will receive
+    // when they are binned from this process.
+    int* send_amounts_per_proc =
+        get_send_amounts_per_proc(rand_nums, numbers_per_proc, world_size);
+
+    // Determine how many numbers you will receive from each process. This
+    // information is needed to set up the binning call.
+    int* recv_amounts_per_proc =
+        get_recv_amounts_per_proc(send_amounts_per_proc, world_size);
+
+    // Do a prefix sum for the send/recv amounts to get the send/recv offsets for
+    // the MPI_Alltoallv call (the binning call).
+    int* send_offsets_per_proc = prefix_sum(send_amounts_per_proc, world_size);
+    int* recv_offsets_per_proc = prefix_sum(recv_amounts_per_proc, world_size);
+
+    // Allocate an array to hold the binned numbers for this process based on the total
+    // amount of numbers this process will receive from others.
+    int    total_recv_amount = sum(recv_amounts_per_proc, world_size);
+    float* binned_nums       = (float*) malloc(sizeof(float) * total_recv_amount);
+
+    // The final step before binning - arrange all of the random numbers so that they
+    // are ordered by bin. For simplicity, we are simply going to sort the random
+    // numbers, however, this could be optimized since the numbers don't need to be
+    // fully sorted.
+    qsort(rand_nums, numbers_per_proc, sizeof(float), &compare_float);
+
+    // Perform the binning step with MPI_Alltoallv. This will send all of the numbers in
+    // the rand_nums array to their proper bin. Each process will only contain numbers
+    // belonging to its bin after this step. For example, if there are 4 processes,
+    // process 0 will contain numbers in the [0, .25) range.
+    MPI_Alltoallv(rand_nums, send_amounts_per_proc, send_offsets_per_proc, MPI_FLOAT,
+                  binned_nums, recv_amounts_per_proc, recv_offsets_per_proc, MPI_FLOAT,
+                  MPI_COMM_WORLD);
+
+    // Print results
+    printf("Process %d received %d numbers in bin [%f - %f)\n", world_rank,
+           total_recv_amount, get_bin_start(world_rank, world_size),
+           get_bin_end(world_rank, world_size));
+
+    // Check that the bin numbers are correct
+    verify_bin_nums(binned_nums, total_recv_amount, world_rank, world_size);
+
+    MPI_Barrier(MPI_COMM_WORLD);
+    MPI_Finalize();
+
+    // Clean up
+    free(rand_nums);
+    free(send_amounts_per_proc);
+    free(recv_amounts_per_proc);
+    free(send_offsets_per_proc);
+    free(recv_offsets_per_proc);
+    free(binned_nums);
+}
@@ -0,0 +1,107 @@
+// Author: Wes Kendall
+// Copyright 2012 www.mpitutorial.com
+// This code is provided freely with the tutorials on mpitutorial.com. Feel
+// free to modify it for your own use. Any distribution of the code must
+// either provide a link to www.mpitutorial.com or keep this header intact.
+//
+// Program that computes the average of an array of elements in parallel using
+// MPI_Scatter and MPI_Allgather
+//
+#include <assert.h>
+#include <mpi.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <time.h>
+
+// Creates an array of random numbers. Each number has a value from 0 - 1
+float*
+create_rand_nums(int num_elements)
+{
+    float* rand_nums = (float*) malloc(sizeof(float) * num_elements);
+    assert(rand_nums != NULL);
+    int i;
+    for(i = 0; i < num_elements; i++)
+    {
+        rand_nums[i] = (rand() / (float) RAND_MAX);
+    }
+    return rand_nums;
+}
+
+// Computes the average of an array of numbers
+float
+compute_avg(float* array, int num_elements)
+{
+    float sum = 0.f;
+    int   i;
+    for(i = 0; i < num_elements; i++)
+    {
+        sum += array[i];
+    }
+    return sum / num_elements;
+}
+
+int
+main(int argc, char** argv)
+{
+    if(argc != 2)
+    {
+        fprintf(stderr, "Usage: avg num_elements_per_proc\n");
+        exit(1);
+    }
+
+    int num_elements_per_proc = atoi(argv[1]);
+    // Seed the random number generator to get different results each time
+    srand(time(NULL));
+
+    MPI_Init(NULL, NULL);
+
+    int world_rank;
+    MPI_Comm_rank(MPI_COMM_WORLD, &world_rank);
+    int world_size;
+    MPI_Comm_size(MPI_COMM_WORLD, &world_size);
+
+    // Create a random array of elements on the root process. Its total
+    // size will be the number of elements per process times the number
+    // of processes
+    float* rand_nums = NULL;
+    if(world_rank == 0)
+    {
+        rand_nums = create_rand_nums(num_elements_per_proc * world_size);
+    }
+
+    // For each process, create a buffer that will hold a subset of the entire
+    // array
+    float* sub_rand_nums = (float*) malloc(sizeof(float) * num_elements_per_proc);
+    assert(sub_rand_nums != NULL);
+
+    // Scatter the random numbers from the root process to all processes in
+    // the MPI world
+    MPI_Scatter(rand_nums, num_elements_per_proc, MPI_FLOAT, sub_rand_nums,
+                num_elements_per_proc, MPI_FLOAT, 0, MPI_COMM_WORLD);
+
+    // Compute the average of your subset
+    float sub_avg = compute_avg(sub_rand_nums, num_elements_per_proc);
+
+    // Gather all partial averages down to all the processes
+    float* sub_avgs = (float*) malloc(sizeof(float) * world_size);
+    assert(sub_avgs != NULL);
+    MPI_Allgather(&sub_avg, 1, MPI_FLOAT, sub_avgs, 1, MPI_FLOAT, MPI_COMM_WORLD);
+
+    // Now that we have all of the partial averages, compute the
+    // total average of all numbers. Since we are assuming each process computed
+    // an average across an equal amount of elements, this computation will
+    // produce the correct answer.
+    float avg = compute_avg(sub_avgs, world_size);
+    printf("Avg of all elements from proc %d is %f\n", world_rank, avg);
+
+    // Clean up
+    if(world_rank == 0)
+    {
+        free(rand_nums);
+    }
+    free(sub_avgs);
+    free(sub_rand_nums);
+
+    MPI_Barrier(MPI_COMM_WORLD);
+    MPI_Finalize();
+}
@@ -0,0 +1,94 @@
+// Author: Wes Kendall
+// Copyright 2013 www.mpitutorial.com
+// This code is provided freely with the tutorials on mpitutorial.com. Feel
+// free to modify it for your own use. Any distribution of the code must
+// either provide a link to www.mpitutorial.com or keep this header intact.
+//
+// Program that computes the standard deviation of an array of elements in parallel using
+// MPI_Reduce.
+//
+#include <assert.h>
+#include <math.h>
+#include <mpi.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <time.h>
+
+// Creates an array of random numbers. Each number has a value from 0 - 1
+float*
+create_rand_nums(int num_elements)
+{
+    float* rand_nums = (float*) malloc(sizeof(float) * num_elements);
+    assert(rand_nums != NULL);
+    int i;
+    for(i = 0; i < num_elements; i++)
+    {
+        rand_nums[i] = (rand() / (float) RAND_MAX);
+    }
+    return rand_nums;
+}
+
+int
+main(int argc, char** argv)
+{
+    if(argc != 2)
+    {
+        fprintf(stderr, "Usage: avg num_elements_per_proc\n");
+        exit(1);
+    }
+
+    int num_elements_per_proc = atoi(argv[1]);
+
+    MPI_Init(NULL, NULL);
+
+    int world_rank;
+    MPI_Comm_rank(MPI_COMM_WORLD, &world_rank);
+    int world_size;
+    MPI_Comm_size(MPI_COMM_WORLD, &world_size);
+
+    // Create a random array of elements on all processes.
+    srand(time(NULL) *
+          world_rank);  // Seed the random number generator of processes uniquely
+    float* rand_nums = NULL;
+    rand_nums        = create_rand_nums(num_elements_per_proc);
+
+    // Sum the numbers locally
+    float local_sum = 0;
+    int   i;
+    for(i = 0; i < num_elements_per_proc; i++)
+    {
+        local_sum += rand_nums[i];
+    }
+
+    // Reduce all of the local sums into the global sum in order to
+    // calculate the mean
+    float global_sum;
+    MPI_Allreduce(&local_sum, &global_sum, 1, MPI_FLOAT, MPI_SUM, MPI_COMM_WORLD);
+    float mean = global_sum / (num_elements_per_proc * world_size);
+
+    // Compute the local sum of the squared differences from the mean
+    float local_sq_diff = 0;
+    for(i = 0; i < num_elements_per_proc; i++)
+    {
+        local_sq_diff += (rand_nums[i] - mean) * (rand_nums[i] - mean);
+    }
+
+    // Reduce the global sum of the squared differences to the root process
+    // and print off the answer
+    float global_sq_diff;
+    MPI_Reduce(&local_sq_diff, &global_sq_diff, 1, MPI_FLOAT, MPI_SUM, 0, MPI_COMM_WORLD);
+
+    // The standard deviation is the square root of the mean of the squared
+    // differences.
+    if(world_rank == 0)
+    {
+        float stddev = sqrt(global_sq_diff / (num_elements_per_proc * world_size));
+        printf("Mean - %f, Standard deviation = %f\n", mean, stddev);
+    }
+
+    // Clean up
+    free(rand_nums);
+
+    MPI_Barrier(MPI_COMM_WORLD);
+    MPI_Finalize();
+}
@@ -0,0 +1,134 @@
+// Author: Wes Kendall
+// Copyright 2011 www.mpitutorial.com
+// This code is provided freely with the tutorials on mpitutorial.com. Feel
+// free to modify it for your own use. Any distribution of the code must
+// either provide a link to www.mpitutorial.com or keep this header intact.
+//
+// Comparison of MPI_Bcast with the my_bcast function
+//
+#include <assert.h>
+#include <mpi.h>
+#include <stdio.h>
+#include <stdlib.h>
+
+void
+my_bcast(void* data, int count, MPI_Datatype datatype, int root, MPI_Comm communicator)
+{
+    int world_rank;
+    MPI_Comm_rank(communicator, &world_rank);
+    int world_size;
+    MPI_Comm_size(communicator, &world_size);
+
+    if(world_rank == root)
+    {
+        // If we are the root process, send our data to everyone
+        int i;
+        for(i = 0; i < world_size; i++)
+        {
+            if(i != world_rank)
+            {
+                MPI_Send(data, count, datatype, i, 0, communicator);
+            }
+        }
+    }
+    else
+    {
+        // If we are a receiver process, receive the data from the root
+        MPI_Recv(data, count, datatype, root, 0, communicator, MPI_STATUS_IGNORE);
+    }
+}
+
+void
+my_ibcast(void* data, int count, MPI_Datatype datatype, int root, MPI_Comm communicator)
+{
+    int world_rank;
+    MPI_Comm_rank(communicator, &world_rank);
+    int world_size;
+    MPI_Comm_size(communicator, &world_size);
+    MPI_Request request = MPI_REQUEST_NULL;
+
+    if(world_rank == root)
+    {
+        // If we are the root process, send our data to everyone
+        int i;
+        for(i = 0; i < world_size; i++)
+        {
+            if(i != world_rank)
+            {
+                MPI_Isend(data, count, datatype, i, 0, communicator, &request);
+            }
+        }
+    }
+    else
+    {
+        // If we are a receiver process, receive the data from the root
+        MPI_Irecv(data, count, datatype, root, 0, communicator, &request);
+    }
+
+    MPI_Status status;
+    // bloks and waits for destination process to receive data
+    MPI_Wait(&request, &status);
+}
+
+int
+main(int argc, char** argv)
+{
+    int num_elements = 30;
+    int num_trials   = 50;
+
+    if(argc != 3) fprintf(stderr, "Usage: compare_bcast [num_elements] [num_trials]\n");
+
+    if(argc > 1) num_elements = atoi(argv[1]);
+    if(argc > 2) num_trials = atoi(argv[2]);
+
+    MPI_Init(NULL, NULL);
+
+    int world_rank;
+    MPI_Comm_rank(MPI_COMM_WORLD, &world_rank);
+
+    double total_my_bcast_time  = 0.0;
+    double total_my_ibcast_time = 0.0;
+    double total_mpi_bcast_time = 0.0;
+    int    i;
+    int*   data = (int*) malloc(sizeof(int) * num_elements);
+    assert(data != NULL);
+
+    for(i = 0; i < num_trials; i++)
+    {
+        // Time my_bcast
+        // Synchronize before starting timing
+        MPI_Barrier(MPI_COMM_WORLD);
+        total_my_bcast_time -= MPI_Wtime();
+        my_bcast(data, num_elements, MPI_INT, 0, MPI_COMM_WORLD);
+        // Synchronize again before obtaining final time
+        MPI_Barrier(MPI_COMM_WORLD);
+        total_my_bcast_time += MPI_Wtime();
+
+        MPI_Barrier(MPI_COMM_WORLD);
+        total_my_ibcast_time -= MPI_Wtime();
+        my_ibcast(data, num_elements, MPI_INT, 0, MPI_COMM_WORLD);
+        // Synchronize again before obtaining final time
+        MPI_Barrier(MPI_COMM_WORLD);
+        total_my_ibcast_time += MPI_Wtime();
+
+        // Time MPI_Bcast
+        MPI_Barrier(MPI_COMM_WORLD);
+        total_mpi_bcast_time -= MPI_Wtime();
+        MPI_Bcast(data, num_elements, MPI_INT, 0, MPI_COMM_WORLD);
+        MPI_Barrier(MPI_COMM_WORLD);
+        total_mpi_bcast_time += MPI_Wtime();
+    }
+
+    // Print off timing information
+    if(world_rank == 0)
+    {
+        printf("Data size = %d, Trials = %d\n", num_elements * (int) sizeof(int),
+               num_trials);
+        printf("Avg my_bcast  time = %lf\n", total_my_bcast_time / num_trials);
+        printf("Avg my_ibcast time = %lf\n", total_my_ibcast_time / num_trials);
+        printf("Avg MPI_Bcast time = %lf\n", total_mpi_bcast_time / num_trials);
+    }
+
+    free(data);
+    MPI_Finalize();
+}
@@ -0,0 +1,82 @@
+// Author: Wes Kendall
+// Copyright 2013 www.mpitutorial.com
+// This code is provided freely with the tutorials on mpitutorial.com. Feel
+// free to modify it for your own use. Any distribution of the code must
+// either provide a link to www.mpitutorial.com or keep this header intact.
+//
+// Program that computes the average of an array of elements in parallel using
+// MPI_Reduce.
+//
+#include <assert.h>
+#include <mpi.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <time.h>
+
+// Creates an array of random numbers. Each number has a value from 0 - 1
+float*
+create_rand_nums(int num_elements)
+{
+    float* rand_nums = (float*) malloc(sizeof(float) * num_elements);
+    assert(rand_nums != NULL);
+    int i;
+    for(i = 0; i < num_elements; i++)
+    {
+        rand_nums[i] = (rand() / (float) RAND_MAX);
+    }
+    return rand_nums;
+}
+
+int
+main(int argc, char** argv)
+{
+    if(argc != 2)
+    {
+        fprintf(stderr, "Usage: avg num_elements_per_proc\n");
+        exit(1);
+    }
+
+    int num_elements_per_proc = atoi(argv[1]);
+
+    MPI_Init(NULL, NULL);
+
+    int world_rank;
+    MPI_Comm_rank(MPI_COMM_WORLD, &world_rank);
+    int world_size;
+    MPI_Comm_size(MPI_COMM_WORLD, &world_size);
+
+    // Create a random array of elements on all processes.
+    srand(time(NULL) * world_rank);  // Seed the random number generator to get different
+                                     // results each time for each processor
+    float* rand_nums = NULL;
+    rand_nums        = create_rand_nums(num_elements_per_proc);
+
+    // Sum the numbers locally
+    float local_sum = 0;
+    int   i;
+    for(i = 0; i < num_elements_per_proc; i++)
+    {
+        local_sum += rand_nums[i];
+    }
+
+    // Print the random numbers on each process
+    printf("Local sum for process %d - %f, avg = %f\n", world_rank, local_sum,
+           local_sum / num_elements_per_proc);
+
+    // Reduce all of the local sums into the global sum
+    float global_sum;
+    MPI_Reduce(&local_sum, &global_sum, 1, MPI_FLOAT, MPI_SUM, 0, MPI_COMM_WORLD);
+
+    // Print the result
+    if(world_rank == 0)
+    {
+        printf("Total sum = %f, avg = %f\n", global_sum,
+               global_sum / (world_size * num_elements_per_proc));
+    }
+
+    // Clean up
+    free(rand_nums);
+
+    MPI_Barrier(MPI_COMM_WORLD);
+    MPI_Finalize();
+}
@@ -0,0 +1,118 @@
+// Author: Wes Kendall
+// Copyright 2012 www.mpitutorial.com
+// This code is provided freely with the tutorials on mpitutorial.com. Feel
+// free to modify it for your own use. Any distribution of the code must
+// either provide a link to www.mpitutorial.com or keep this header intact.
+//
+// Program that computes the average of an array of elements in parallel using
+// MPI_Scatter and MPI_Gather
+//
+#include <assert.h>
+#include <mpi.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <time.h>
+
+// Creates an array of random numbers. Each number has a value from 0 - 1
+float*
+create_rand_nums(int num_elements)
+{
+    float* rand_nums = (float*) malloc(sizeof(float) * num_elements);
+    assert(rand_nums != NULL);
+    int i;
+    for(i = 0; i < num_elements; i++)
+    {
+        rand_nums[i] = (rand() / (float) RAND_MAX);
+    }
+    return rand_nums;
+}
+
+// Computes the average of an array of numbers
+float
+compute_avg(float* array, int num_elements)
+{
+    float sum = 0.f;
+    int   i;
+    for(i = 0; i < num_elements; i++)
+    {
+        sum += array[i];
+    }
+    return sum / num_elements;
+}
+
+int
+main(int argc, char** argv)
+{
+    if(argc != 2)
+    {
+        fprintf(stderr, "Usage: avg num_elements_per_proc\n");
+        exit(1);
+    }
+
+    int num_elements_per_proc = atoi(argv[1]);
+    // Seed the random number generator to get different results each time
+    srand(time(NULL));
+
+    MPI_Init(NULL, NULL);
+
+    int world_rank;
+    MPI_Comm_rank(MPI_COMM_WORLD, &world_rank);
+    int world_size;
+    MPI_Comm_size(MPI_COMM_WORLD, &world_size);
+
+    // Create a random array of elements on the root process. Its total
+    // size will be the number of elements per process times the number
+    // of processes
+    float* rand_nums = NULL;
+    if(world_rank == 0)
+    {
+        rand_nums = create_rand_nums(num_elements_per_proc * world_size);
+    }
+
+    // For each process, create a buffer that will hold a subset of the entire
+    // array
+    float* sub_rand_nums = (float*) malloc(sizeof(float) * num_elements_per_proc);
+    assert(sub_rand_nums != NULL);
+
+    // Scatter the random numbers from the root process to all processes in
+    // the MPI world
+    MPI_Scatter(rand_nums, num_elements_per_proc, MPI_FLOAT, sub_rand_nums,
+                num_elements_per_proc, MPI_FLOAT, 0, MPI_COMM_WORLD);
+
+    // Compute the average of your subset
+    float sub_avg = compute_avg(sub_rand_nums, num_elements_per_proc);
+
+    // Gather all partial averages down to the root process
+    float* sub_avgs = NULL;
+    if(world_rank == 0)
+    {
+        sub_avgs = (float*) malloc(sizeof(float) * world_size);
+        assert(sub_avgs != NULL);
+    }
+    MPI_Gather(&sub_avg, 1, MPI_FLOAT, sub_avgs, 1, MPI_FLOAT, 0, MPI_COMM_WORLD);
+
+    // Now that we have all of the partial averages on the root, compute the
+    // total average of all numbers. Since we are assuming each process computed
+    // an average across an equal amount of elements, this computation will
+    // produce the correct answer.
+    if(world_rank == 0)
+    {
+        float avg = compute_avg(sub_avgs, world_size);
+        printf("Avg of all elements is %f\n", avg);
+        // Compute the average across the original data for comparison
+        float original_data_avg =
+            compute_avg(rand_nums, num_elements_per_proc * world_size);
+        printf("Avg computed across original data is %f\n", original_data_avg);
+    }
+
+    // Clean up
+    if(world_rank == 0)
+    {
+        free(rand_nums);
+        free(sub_avgs);
+    }
+    free(sub_rand_nums);
+
+    MPI_Barrier(MPI_COMM_WORLD);
+    MPI_Finalize();
+}
@@ -0,0 +1,55 @@
+// Author: Wes Kendall
+// Copyright 2011 www.mpitutorial.com
+// This code is provided freely with the tutorials on mpitutorial.com. Feel
+// free to modify it for your own use. Any distribution of the code must
+// either provide a link to www.mpitutorial.com or keep this header intact.
+//
+// Ping pong example with MPI_Send and MPI_Recv. Two processes ping pong a
+// number back and forth, incrementing it until it reaches a given value.
+//
+#include <mpi.h>
+#include <stdio.h>
+#include <stdlib.h>
+
+int
+main(int argc, char** argv)
+{
+    const int PING_PONG_LIMIT = 10;
+
+    // Initialize the MPI environment
+    MPI_Init(NULL, NULL);
+    // Find out rank, size
+    int world_rank;
+    MPI_Comm_rank(MPI_COMM_WORLD, &world_rank);
+    int world_size;
+    MPI_Comm_size(MPI_COMM_WORLD, &world_size);
+
+    // We are assuming 2 processes for this task
+    if(world_size != 2)
+    {
+        fprintf(stderr, "World size must be two for %s\n", argv[0]);
+        MPI_Abort(MPI_COMM_WORLD, 1);
+    }
+
+    int ping_pong_count = 0;
+    int partner_rank    = (world_rank + 1) % 2;
+    while(ping_pong_count < PING_PONG_LIMIT)
+    {
+        if(world_rank == ping_pong_count % 2)
+        {
+            // Increment the ping pong count before you send it
+            ping_pong_count++;
+            MPI_Send(&ping_pong_count, 1, MPI_INT, partner_rank, 0, MPI_COMM_WORLD);
+            printf("%d sent and incremented ping_pong_count %d to %d\n", world_rank,
+                   ping_pong_count, partner_rank);
+        }
+        else
+        {
+            MPI_Recv(&ping_pong_count, 1, MPI_INT, partner_rank, 0, MPI_COMM_WORLD,
+                     MPI_STATUS_IGNORE);
+            printf("%d received ping_pong_count %d from %d\n", world_rank,
+                   ping_pong_count, partner_rank);
+        }
+    }
+    MPI_Finalize();
+}