Use finegrain allocator by default (#140)

* Use FineGrained allocator for heap by default, consolidate all types of
allocators under saner cmake controls

Co-authored-by: Yiltan <ytemucin@amd.com>

* Uncached may not be only for debug

Need to include the rocshmem config otherwise produce an inconsistent
build with different allocators used in different files

* Undo this pr adding presumably useless hip_host_allocator_noncoherent

* Rename HEAP_IS_COHERENT/USE_COHERENT_HEAP to USE_HDP_FLUSH as the former
was misleading

* Remove unused __roc_inv()

---------

Co-authored-by: Yiltan <ytemucin@amd.com>

src/host/host.cpp

@@ -118,16 +118,16 @@ __host__ HostInterface::HostInterface(HdpPolicy* hdp_policy,
         new HostContextWindowInfo(host_comm_world_, heap);
   }
 
-#if !defined(USE_COHERENT_HEAP) && !defined(USE_SINGLE_NODE)
+#if defined(USE_HDP_FLUSH) && !defined(USE_SINGLE_NODE)
   // The single node implementation needs a different path since
   // the HDP flush pointers are allocated on the symmetric heap
   // and we need to wait for other initialization to happen before
   // calling `get_hdp_flush_ptr`.
   create_hdp_window();
-#endif  // defined(USE_COHERENT_HEAP) && !defined(USE_SINGLE_NODE)
+#endif  // defined(USE_HDP_FLUSH) && !defined(USE_SINGLE_NODE)
 }
 
-#ifndef USE_COHERENT_HEAP
+#if defined USE_HDP_FLUSH
 __host__ void HostInterface::create_hdp_window() {
   MPI_Win_create(hdp_policy_->get_hdp_flush_ptr(),
                  sizeof(unsigned int), /* size of window */
@@ -142,14 +142,14 @@ __host__ void HostInterface::create_hdp_window() {
    */
   MPI_Win_lock_all(MPI_MODE_NOCHECK, hdp_win);
 }
-#endif  // USE_COHERENT_HEAP
+#endif  // USE_HDP_FLUSH
 
 __host__ HostInterface::~HostInterface() {
-#ifndef USE_COHERENT_HEAP
+#if defined USE_HDP_FLUSH
   MPI_Win_unlock_all(hdp_win);
 
   MPI_Win_free(&hdp_win);
-#endif  // USE_COHERENT_HEAP
+#endif  // USE_HDP_FLUSH
 
   /* Detroy the pool of contexts */
   for (int ctx_i = 0; ctx_i < max_num_ctxs_; ctx_i++) {

src/host/host.hpp

@@ -248,16 +248,16 @@ class HostInterface {
   template <typename T>
   __host__ int test(T *ivars, int cmp, T val, WindowInfo* window_info);
 
-#ifndef USE_COHERENT_HEAP
+#if defined USE_HDP_FLUSH
   __host__ void create_hdp_window();
-#endif // USE_COHERENT_HEAP
+#endif // USE_HDP_FLUSH
 
  private:
   /**************************************************************************
    **************************** INTERNAL METHODS ****************************
    *************************************************************************/
   __host__ void flush_remote_hdps() {
-#ifndef USE_COHERENT_HEAP
+#if defined USE_HDP_FLUSH
     unsigned flush_val{HdpPolicy::HDP_FLUSH_VAL};
     for (size_t i{0}; i < num_pes_; i++) {
       if (i == my_pe_) {
@@ -266,15 +266,15 @@ class HostInterface {
       MPI_Put(&flush_val, 1, MPI_UNSIGNED, i, 0, 1, MPI_UNSIGNED, hdp_win);
     }
     MPI_Win_flush_all(hdp_win);
-#endif // USE_COHERENT_HEAP
+#endif // USE_HDP_FLUSH
   }
 
   __host__ void flush_remote_hdp(int pe) {
-#ifndef USE_COHERENT_HEAP
+#if defined USE_HDP_FLUSH
     unsigned flush_val{HdpPolicy::HDP_FLUSH_VAL};
     MPI_Put(&flush_val, 1, MPI_UNSIGNED, pe, 0, 1, MPI_UNSIGNED, hdp_win);
     MPI_Win_flush(pe, hdp_win);
-#endif // USE_COHERENT_HEAP
+#endif // USE_HDP_FLUSH
   }
 
   __host__ void initiate_put(void* dest, const void* source, size_t nelems,
@@ -333,12 +333,12 @@ class HostInterface {
    */
   int num_pes_{0};
 
-#ifndef USE_COHERENT_HEAP
+#if defined USE_HDP_FLUSH
   /**
    * @brief MPI window for hdp flushing
    */
   MPI_Win hdp_win;
-#endif  // USE_COHERENT_HEAP
+#endif  // USE_HDP_FLUSH
 
   /**
    * @brief Max number of contexts for the application