diff --git a/hipamd/src/hip_graph.cpp b/hipamd/src/hip_graph.cpp
index f3a93253cf..fddc49b020 100644
--- a/hipamd/src/hip_graph.cpp
+++ b/hipamd/src/hip_graph.cpp
@@ -2297,6 +2297,10 @@ hipError_t hipUserObjectRelease(hipUserObject_t object, unsigned int count) {
   if (object->referenceCount() < count || !hipUserObject::isUserObjvalid(object)) {
     HIP_RETURN(hipSuccess);
   }
+  //! If all the counts are gone not longer need the obj in the list
+  if (object->referenceCount() == count) {
+    hipUserObject::removeUSerObj(object);
+  }
   object->decreaseRefCount(count);
   HIP_RETURN(hipSuccess);
 }
diff --git a/hipamd/src/hip_graph_internal.hpp b/hipamd/src/hip_graph_internal.hpp
index 38f72581c0..819125c1f3 100644
--- a/hipamd/src/hip_graph_internal.hpp
+++ b/hipamd/src/hip_graph_internal.hpp
@@ -76,8 +76,8 @@ struct hipUserObject : public amd::ReferenceCountedObject {
   }
 
   static bool isUserObjvalid(hipUserObject* pUsertObj) {
-    amd::ScopedLock lock(UserObjectLock_);
-    if (ObjectSet_.find(pUsertObj) == ObjectSet_.end()) {
+    auto it = ObjectSet_.find(pUsertObj);
+    if (it == ObjectSet_.end()) {
       return false;
     }
     return true;
@@ -85,8 +85,9 @@ struct hipUserObject : public amd::ReferenceCountedObject {
 
   static void removeUSerObj(hipUserObject* pUsertObj) {
     amd::ScopedLock lock(UserObjectLock_);
-    if (ObjectSet_.find(pUsertObj) == ObjectSet_.end()) {
-      ObjectSet_.erase(pUsertObj);
+    auto it = ObjectSet_.find(pUsertObj);
+    if (it != ObjectSet_.end()) {
+      ObjectSet_.erase(it);
     }
   }