diff --git a/projects/clr/hipamd/src/hip_graph_internal.hpp b/projects/clr/hipamd/src/hip_graph_internal.hpp
index 5bbbd521b8..a2fafdb35c 100644
--- a/projects/clr/hipamd/src/hip_graph_internal.hpp
+++ b/projects/clr/hipamd/src/hip_graph_internal.hpp
@@ -2328,10 +2328,6 @@ class GraphMemAllocNode final : public GraphNode {
       memory_ = getMemoryObject(dptr, offset);
       // Retain memory object because command release will release it
       memory_->retain();
-
-      // Remove because the entry is not needed in MemObjMap after the memory_ has been saved.
-      // The Phy mem obj will be saved in virtual memory object during VirtualMapCommand::submit.
-      amd::MemObjMap::RemoveMemObj(dptr);
       size_ = aligned_size;
       // Execute the original mapping command
       VirtualMapCommand::submit(device);
diff --git a/projects/clr/hipamd/src/hip_mempool_impl.cpp b/projects/clr/hipamd/src/hip_mempool_impl.cpp
index 62076cf91b..5c180ced08 100644
--- a/projects/clr/hipamd/src/hip_mempool_impl.cpp
+++ b/projects/clr/hipamd/src/hip_mempool_impl.cpp
@@ -226,8 +226,6 @@ void* MemoryPool::AllocateMemory(size_t size, Stream* stream, void* dptr) {
     }
   } else {
     dev_ptr = memory->getSvmPtr();
-    if (!amd::MemObjMap::FindMemObj(dev_ptr))
-      amd::MemObjMap::AddMemObj(dev_ptr, memory);
   }
   // Place the allocated memory into the busy heap
   ts.AddSafeStream(stream);