Use finegrain allocator by default (#140)

* Use FineGrained allocator for heap by default, consolidate all types of
allocators under saner cmake controls

Co-authored-by: Yiltan <ytemucin@amd.com>

* Uncached may not be only for debug

Need to include the rocshmem config otherwise produce an inconsistent
build with different allocators used in different files

* Undo this pr adding presumably useless hip_host_allocator_noncoherent

* Rename HEAP_IS_COHERENT/USE_COHERENT_HEAP to USE_HDP_FLUSH as the former
was misleading

* Remove unused __roc_inv()

---------

Co-authored-by: Yiltan <ytemucin@amd.com>

[ROCm/rocshmem commit: 41fd9e2d57]

projects/rocshmem/src/assembly.hpp

@@ -165,23 +165,8 @@ NOWARN(-Wdeprecated-volatile,
 )
 // clang-format on
 
-__device__ __forceinline__ void __roc_inv() {
-#if defined USE_COHERENT_HEAP
-#if defined(__gfx906__)
-#endif
-#if defined(__gfx908__)
-#endif
-#if defined(__gfx90a__)
-//  asm volatile("buffer_wbinvl1;");
-#endif
-#if defined(__gfx942__)
-//  asm volatile("buffer_inv sc0 sc1;");
-#endif
-#endif
-}
-
 __device__ __forceinline__ void __roc_flush() {
-#if defined USE_COHERENT_HEAP
+#if not defined USE_HDP_FLUSH
 #if defined(__gfx906__)
 #endif
 #if defined(__gfx908__)