Add a custom allreduce algorithm in MSCCLPP for cpx mode (#1362)

* cmake: remove mscclpp patch after build is complete

To enable mscclpp in cpx mode, a patch cpx.patch needs to be applied.
This patch can be removed after building is done. This helps with the
build process the following time.

* Use read-based mscclpp allreduce from rccl

MSCCLPP by default uses remote write in the allreduce kernel for
large (> 1MB) messages. This PR adds an allreduce kernel that uses
remote read. It needs the users to use an environment variable
MSCCLPP_READ_ALLRED=1.

cmake/MSCCLPP.cmake

@@ -66,7 +66,12 @@ if(ENABLE_MSCCLPP)
         execute_process(
            COMMAND git apply ${CMAKE_CURRENT_SOURCE_DIR}/ext-src/cpx.patch
            WORKING_DIRECTORY ${MSCCLPP_SOURCE}
-        ) 
+        )
+	execute_process(
+           COMMAND git apply ${CMAKE_CURRENT_SOURCE_DIR}/ext-src/read-allred.patch
+           WORKING_DIRECTORY ${MSCCLPP_SOURCE}
+        )
+
         message(STATUS "Building mscclpp only for gfx942.")
         
         mscclpp_cmake_arg(CMAKE_PREFIX_PATH)
@@ -89,6 +94,14 @@ if(ENABLE_MSCCLPP)
         )
 
         find_package(mscclpp_nccl REQUIRED)
+	execute_process(
+    		COMMAND git apply --reverse ${CMAKE_CURRENT_SOURCE_DIR}/ext-src/cpx.patch
+        	WORKING_DIRECTORY ${MSCCLPP_SOURCE}
+    	)
+	execute_process(
+           COMMAND git apply --reverse ${CMAKE_CURRENT_SOURCE_DIR}/ext-src/read-allred.patch
+           WORKING_DIRECTORY ${MSCCLPP_SOURCE}
+        )
     endif()
 
     execute_process(COMMAND objcopy
@@ -98,4 +111,5 @@ if(ENABLE_MSCCLPP)
     )
     add_library(mscclpp_nccl STATIC IMPORTED)
     set_target_properties(mscclpp_nccl PROPERTIES IMPORTED_LOCATION ${PROJECT_BINARY_DIR}/libmscclpp_nccl.a)
+
 endif()