diff --git a/include/libtorrent/kademlia/rpc_manager.hpp b/include/libtorrent/kademlia/rpc_manager.hpp
index 559e92095..bb155d0a2 100644
--- a/include/libtorrent/kademlia/rpc_manager.hpp
+++ b/include/libtorrent/kademlia/rpc_manager.hpp
@@ -104,6 +104,7 @@ public:
 
 		auto deleter = [this](observer* o)
 		{
+			TORRENT_ASSERT(o->m_in_use);
 			o->~observer();
 			free_observer(o);
 		};
diff --git a/src/kademlia/rpc_manager.cpp b/src/kademlia/rpc_manager.cpp
index 17e8bd583..29ebffdff 100644
--- a/src/kademlia/rpc_manager.cpp
+++ b/src/kademlia/rpc_manager.cpp
@@ -200,7 +200,6 @@ void rpc_manager::free_observer(void* ptr)
 {
 	if (ptr == nullptr) return;
 	--m_allocated_observers;
-	TORRENT_ASSERT(reinterpret_cast<observer*>(ptr)->m_in_use == false);
 	m_pool_allocator.free(ptr);
 }