Share the DHT storage for both IPv4 and IPv6 (#762)

.gitignore

@@ -44,6 +44,7 @@ libtool
 
 .DS_Store
 .idea
+*~
 
 # Compile and link flag files
 bindings/python/*flags

include/libtorrent/aux_/session_impl.hpp

@@ -991,6 +991,7 @@ namespace libtorrent
 			mutable int m_next_port;
 
 #ifndef TORRENT_DISABLE_DHT
+			boost::shared_ptr<dht::dht_storage_interface> m_dht_storage;
 			boost::shared_ptr<dht::dht_tracker> m_dht;
 			dht_settings m_dht_settings;
 			dht::dht_storage_constructor_type m_dht_storage_constructor;

include/libtorrent/kademlia/dht_storage.hpp

@@ -39,6 +39,8 @@ POSSIBILITY OF SUCH DAMAGE.
 
 #include "libtorrent/aux_/disable_warnings_pop.hpp"
 
+#include <libtorrent/kademlia/node_id.hpp>
+
 #include <libtorrent/socket.hpp>
 #include <libtorrent/sha1_hash.hpp>
 #include <libtorrent/address.hpp>
@@ -59,6 +61,9 @@ namespace dht
 		boost::int32_t peers;
 		boost::int32_t immutable_data;
 		boost::int32_t mutable_data;
+
+		// This member function set the counters to zero.
+		void reset();
 	};
 
 	// The DHT storage interface is a pure virtual class that can
@@ -88,6 +93,13 @@ namespace dht
 		virtual size_t num_peers() const = 0;
 #endif
 
+		// This member function notifies the list of all node's ids
+		// of each DHT running inside libtorrent. It's advisable
+		// that the concrete implementation keeps a copy of this list
+		// for an eventual prioritization when deleting an element
+		// to make room for a new one.
+		virtual void update_node_ids(std::vector<node_id> const& ids) = 0;
+
 		// This function retrieve the peers tracked by the DHT
 		// corresponding to the given info_hash. You can specify if
 		// you want only seeds and/or you are scraping the data.
@@ -207,11 +219,9 @@ namespace dht
 		virtual ~dht_storage_interface() {}
 	};
 
-	typedef boost::function<dht_storage_interface*(sha1_hash const& id
+	typedef boost::function<dht_storage_interface*(dht_settings const& settings)> dht_storage_constructor_type;
-		, dht_settings const& settings)> dht_storage_constructor_type;
 
-	TORRENT_EXPORT dht_storage_interface* dht_default_storage_constructor(sha1_hash const& id
+	TORRENT_EXPORT dht_storage_interface* dht_default_storage_constructor(dht_settings const& settings);
-		, dht_settings const& settings);
 
 } } // namespace libtorrent::dht
 

include/libtorrent/kademlia/dht_tracker.hpp

@@ -76,8 +76,9 @@ namespace libtorrent { namespace dht
 		dht_tracker(dht_observer* observer
 			, io_service& ios
 			, send_fun_t const& send_fun
-			, dht_settings const& settings, counters& cnt
+			, dht_settings const& settings
-			, dht_storage_constructor_type storage_constructor
+			, counters& cnt
+			, dht_storage_interface& storage
 			, entry const& state);
 		virtual ~dht_tracker();
 
@@ -144,6 +145,7 @@ namespace libtorrent { namespace dht
 		void connection_timeout(node& n, error_code const& e);
 		void refresh_timeout(error_code const& e);
 		void refresh_key(error_code const& e);
+		void update_storage_node_ids();
 
 		// implements udp_socket_interface
 		virtual bool has_quota() override;
@@ -155,6 +157,7 @@ namespace libtorrent { namespace dht
 		bdecode_node m_msg;
 
 		counters& m_counters;
+		dht_storage_interface& m_storage;
 		node m_dht;
 #if TORRENT_USE_IPV6
 		node m_dht6;

include/libtorrent/kademlia/node.hpp

@@ -103,7 +103,7 @@ public:
 		, libtorrent::dht_settings const& settings, node_id nid
 		, dht_observer* observer, counters& cnt
 		, std::map<std::string, node*> const& nodes
-		, dht_storage_constructor_type storage_constructor = dht_default_storage_constructor);
+		, dht_storage_interface& storage);
 
 	~node();
 
@@ -118,8 +118,8 @@ public:
 	void incoming(msg const& m);
 
 #ifndef TORRENT_NO_DEPRECATE
-	int num_torrents() const { return int(m_storage->num_torrents()); }
+	int num_torrents() const { return int(m_storage.num_torrents()); }
-	int num_peers() const { return int(m_storage->num_peers()); }
+	int num_peers() const { return int(m_storage.num_peers()); }
 #endif
 
 	int bucket_size(int bucket);
@@ -131,7 +131,7 @@ public:
 	{ return m_table.num_global_nodes(); }
 
 #ifndef TORRENT_NO_DEPRECATE
-	int data_size() const { return int(m_storage->num_torrents()); }
+	int data_size() const { return int(m_storage.num_torrents()); }
 #endif
 
 #if defined TORRENT_DEBUG
@@ -285,7 +285,7 @@ private:
 	udp_socket_interface* m_sock;
 	counters& m_counters;
 
-	boost::scoped_ptr<dht_storage_interface> m_storage;
+	dht_storage_interface& m_storage;
 };
 
 } } // namespace libtorrent::dht

include/libtorrent/kademlia/node_id.hpp

@@ -33,6 +33,7 @@ POSSIBILITY OF SUCH DAMAGE.
 #define NODE_ID_HPP
 
 #include <algorithm>
+#include <vector>
 
 #include "libtorrent/aux_/disable_warnings_push.hpp"
 #include <boost/cstdint.hpp>
@@ -63,6 +64,7 @@ bool TORRENT_EXTRA_EXPORT compare_ref(node_id const& n1, node_id const& n2, node
 // after the shared bit prefix of ``n1`` and ``n2``.
 // if the first bits are different, that's 160.
 int TORRENT_EXTRA_EXPORT distance_exp(node_id const& n1, node_id const& n2);
+int TORRENT_EXTRA_EXPORT min_distance_exp(node_id const& n1, std::vector<node_id> const& ids);
 
 node_id TORRENT_EXTRA_EXPORT generate_id(address const& external_ip);
 node_id TORRENT_EXTRA_EXPORT generate_random_id();

simulation/setup_dht.cpp

@@ -84,16 +84,17 @@ struct dht_node final : lt::dht::udp_socket_interface
 	dht_node(sim::simulation& sim, lt::dht_settings const& sett, lt::counters& cnt
 		, int idx, std::uint32_t flags)
 		: m_io_service(sim, (flags & dht_network::bind_ipv6) ? addr6_from_int(idx) : addr_from_int(idx))
+		, m_dht_storage(lt::dht::dht_default_storage_constructor(sett))
 #if LIBSIMULATOR_USE_MOVE
 		, m_socket(m_io_service)
 		, m_dht((flags & dht_network::bind_ipv6) ? udp::v6() : udp::v4()
 			, this, sett, id_from_addr(m_io_service.get_ips().front())
-			, nullptr, cnt, std::map<std::string, lt::dht::node*>())
+			, nullptr, cnt, std::map<std::string, lt::dht::node*>(), *m_dht_storage)
 #else
 		, m_socket(new asio::ip::udp::socket(m_io_service))
 		, m_dht(new lt::dht::node((flags & dht_network::bind_ipv6) ? udp::v6() : udp::v4()
 			, this, sett, id_from_addr(m_io_service.get_ips().front())
-			, nullptr, cnt, std::map<std::string, lt::dht::node*>()))
+			, nullptr, cnt, std::map<std::string, lt::dht::node*>(), *m_dht_storage))
 #endif
 		, m_add_dead_nodes(flags & dht_network::add_dead_nodes)
 		, m_ipv6(flags & dht_network::bind_ipv6)
@@ -124,7 +125,7 @@ struct dht_node final : lt::dht::udp_socket_interface
 		: m_socket(std::move(n.m_socket))
 		, m_dht(n.m_ipv6 ? udp::v6() : udp::v4(), this, n.m_dht.settings(), n.m_dht.nid()
 			, n.m_dht.observer(), n.m_dht.stats_counters()
-			, std::map<std::string, lt::dht::node*>())
+			, std::map<std::string, lt::dht::node*>(), *n.m_dht_storage)
 	{
 		assert(false && "dht_node is not movable");
 		throw std::runtime_error("dht_node is not movable");
@@ -252,6 +253,7 @@ struct dht_node final : lt::dht::udp_socket_interface
 
 private:
 	asio::io_service m_io_service;
+	boost::shared_ptr<dht::dht_storage_interface> m_dht_storage;
 #if LIBSIMULATOR_USE_MOVE
 	lt::udp::socket m_socket;
 	lt::udp::socket& sock() { return m_socket; }

simulation/test_dht_rate_limit.cpp

@@ -106,9 +106,10 @@ TORRENT_TEST(dht_rate_limit)
 
 	counters cnt;
 	entry state;
+	boost::scoped_ptr<lt::dht::dht_storage_interface> dht_storage(dht::dht_default_storage_constructor(dhtsett));
 	boost::shared_ptr<lt::dht::dht_tracker> dht = boost::make_shared<lt::dht::dht_tracker>(
 		&o, boost::ref(dht_ios), std::bind(&udp_socket::send, &sock, _1, _2, _3, _4)
-		, dhtsett, cnt, dht::dht_default_storage_constructor, state);
+		, dhtsett, cnt, *dht_storage, state);
 
 	bool stop = false;
 	std::function<void(error_code const&, size_t)> on_read

simulation/test_dht_storage.cpp

@@ -108,7 +108,7 @@ TORRENT_TEST(dht_storage_counters)
 {
 #ifndef TORRENT_DISABLE_DHT
 	dht_settings sett = test_settings();
-	boost::shared_ptr<dht_storage_interface> s(dht_default_storage_constructor(node_id(0), sett));
+	boost::shared_ptr<dht_storage_interface> s(dht_default_storage_constructor(sett));
 
 	TEST_CHECK(s.get() != NULL);
 

src/kademlia/dht_storage.cpp

@@ -59,8 +59,6 @@ POSSIBILITY OF SUCH DAMAGE.
 #include <libtorrent/kademlia/item.hpp>
 #include <libtorrent/kademlia/node_id.hpp>
 
-#include <string.h> // for memset
-
 namespace libtorrent {
 namespace dht {
 namespace
@@ -154,15 +152,15 @@ namespace
 	// less important to keep
 	struct immutable_item_comparator
 	{
-		immutable_item_comparator(node_id const& our_id) : m_our_id(our_id) {}
+		immutable_item_comparator(std::vector<node_id> const& node_ids) : m_node_ids(node_ids) {}
 		immutable_item_comparator(immutable_item_comparator const& c)
-			: m_our_id(c.m_our_id) {}
+			: m_node_ids(c.m_node_ids) {}
 
 		bool operator() (std::pair<node_id, dht_immutable_item> const& lhs
 			, std::pair<node_id, dht_immutable_item> const& rhs) const
 		{
-			int l_distance = distance_exp(lhs.first, m_our_id);
+			int l_distance = min_distance_exp(lhs.first, m_node_ids);
-			int r_distance = distance_exp(rhs.first, m_our_id);
+			int r_distance = min_distance_exp(rhs.first, m_node_ids);
 
 			// this is a score taking the popularity (number of announcers) and the
 			// fit, in terms of distance from ideal storing node, into account.
@@ -178,9 +176,21 @@ namespace
 		// explicitly disallow assignment, to silence msvc warning
 		immutable_item_comparator& operator=(immutable_item_comparator const&);
 
-		node_id const& m_our_id;
+		std::vector<node_id> const& m_node_ids;
 	};
 
+	// picks the least important one (i.e. the one
+	// the fewest peers are announcing, and farthest
+	// from our node IDs)
+	template<class Item>
+	typename std::map<node_id, Item>::const_iterator pick_least_important_item(
+		std::vector<node_id> const& node_ids, std::map<node_id, Item> const& table)
+	{
+		return std::min_element(table.begin()
+			, table.end()
+			, immutable_item_comparator(node_ids));
+	}
+
 	class dht_default_storage final : public dht_storage_interface, boost::noncopyable
 	{
 	typedef std::map<node_id, torrent_entry> table_t;
@@ -189,11 +199,10 @@ namespace
 
 	public:
 
-		dht_default_storage(sha1_hash const& id, dht_settings const& settings)
+		dht_default_storage(dht_settings const& settings)
-			: m_id(id)
+			: m_settings(settings)
-			, m_settings(settings)
 		{
-			memset(&m_counters, 0, sizeof(m_counters));
+			m_counters.reset();
 		}
 
 		~dht_default_storage() {}
@@ -207,6 +216,10 @@ namespace
 			return ret;
 		}
 #endif
+		void update_node_ids(std::vector<node_id> const& ids) override
+		{
+			m_node_ids = ids;
+		}
 
 		bool get_peers(sha1_hash const& info_hash
 			, bool noseed, bool scrape
@@ -355,12 +368,8 @@ namespace
 				// make sure we don't add too many items
 				if (int(m_immutable_table.size()) >= m_settings.max_dht_items)
 				{
-					// delete the least important one (i.e. the one
+					auto j = pick_least_important_item(m_node_ids
-					// the fewest peers are announcing, and farthest
+						, m_immutable_table);
-					// from our node ID)
-					dht_immutable_table_t::iterator j = std::min_element(m_immutable_table.begin()
-						, m_immutable_table.end()
-						, immutable_item_comparator(m_id));
 
 					TORRENT_ASSERT(j != m_immutable_table.end());
 					free(j->second.value);
@@ -425,13 +434,8 @@ namespace
 				// make sure we don't add too many items
 				if (int(m_mutable_table.size()) >= m_settings.max_dht_items)
 				{
-					// delete the least important one (i.e. the one
+					auto j = pick_least_important_item(m_node_ids
-					// the fewest peers are announcing)
+						, m_mutable_table);
-					dht_mutable_table_t::iterator j = std::min_element(m_mutable_table.begin()
-						, m_mutable_table.end()
-						, [] (dht_mutable_table_t::value_type const& lhs
-							, dht_mutable_table_t::value_type const& rhs)
-						{ return lhs.second.num_announcers < rhs.second.num_announcers; });
 
 					TORRENT_ASSERT(j != m_mutable_table.end());
 					free(j->second.value);
@@ -542,10 +546,10 @@ namespace
 		}
 
 	private:
-		sha1_hash m_id;
 		dht_settings const& m_settings;
 		dht_storage_counters m_counters;
 
+		std::vector<node_id> m_node_ids;
 		table_t m_map;
 		dht_immutable_table_t m_immutable_table;
 		dht_mutable_table_t m_mutable_table;
@@ -568,10 +572,17 @@ namespace
 	};
 }
 
-dht_storage_interface* dht_default_storage_constructor(sha1_hash const& id
+void dht_storage_counters::reset()
-	, dht_settings const& settings)
 {
-	return new dht_default_storage(id, settings);
+	torrents = 0;
+	peers = 0;
+	immutable_data = 0;
+	mutable_data = 0;
+}
+
+dht_storage_interface* dht_default_storage_constructor(dht_settings const& settings)
+{
+	return new dht_default_storage(settings);
 }
 
 } } // namespace libtorrent::dht

src/kademlia/dht_tracker.cpp

@@ -98,14 +98,15 @@ namespace libtorrent { namespace dht
 		, send_fun_t const& send_fun
 		, dht_settings const& settings
 		, counters& cnt
-		, dht_storage_constructor_type storage_constructor
+		, dht_storage_interface& storage
 		, entry const& state)
 		: m_counters(cnt)
+		, m_storage(storage)
 		, m_dht(udp::v4(), this, settings, extract_node_id(state, "node-id")
-			, observer, cnt, m_nodes, storage_constructor)
+			, observer, cnt, m_nodes, storage)
 #if TORRENT_USE_IPV6
 		, m_dht6(udp::v6(), this, settings, extract_node_id(state, "node-id6")
-			, observer, cnt, m_nodes, storage_constructor)
+			, observer, cnt, m_nodes, storage)
 #endif
 		, m_send_fun(send_fun)
 		, m_log(observer)
@@ -129,6 +130,8 @@ namespace libtorrent { namespace dht
 		m_nodes.insert(std::make_pair(m_dht6.protocol_family_name(), &m_dht6));
 #endif
 
+		update_storage_node_ids();
+
 #ifndef TORRENT_DISABLE_LOGGING
 		m_log->log(dht_logger::tracker, "starting IPv4 DHT tracker with node id: %s"
 			, to_hex(m_dht.nid().to_string()).c_str());
@@ -144,6 +147,10 @@ namespace libtorrent { namespace dht
 	void dht_tracker::update_node_id()
 	{
 		m_dht.update_node_id();
+#if TORRENT_USE_IPV6
+		m_dht6.update_node_id();
+#endif
+		update_storage_node_ids();
 	}
 
 	// defined in node.cpp
@@ -208,19 +215,19 @@ namespace libtorrent { namespace dht
 #ifndef TORRENT_NO_DEPRECATE
 	void dht_tracker::dht_status(session_status& s)
 	{
-		m_dht.status(s);
+		m_dht.status(s); //TODO: What to do with m_dht6?
 	}
 #endif
 
 	void dht_tracker::dht_status(std::vector<dht_routing_bucket>& table
 		, std::vector<dht_lookup>& requests)
 	{
-		m_dht.status(table, requests);
+		m_dht.status(table, requests); //TODO: What to do with m_dht6?
 	}
 
 	void dht_tracker::update_stats_counters(counters& c) const
 	{
-		m_dht.update_stats_counters(c);
+		m_dht.update_stats_counters(c); //TODO: What to do with m_dht6?
 	}
 
 	void dht_tracker::connection_timeout(node& n, error_code const& e)
@@ -266,12 +273,24 @@ namespace libtorrent { namespace dht
 		m_key_refresh_timer.async_wait(std::bind(&dht_tracker::refresh_key, self(), _1));
 
 		m_dht.new_write_key();
+#if TORRENT_USE_IPV6
 		m_dht6.new_write_key();
+#endif
 #ifndef TORRENT_DISABLE_LOGGING
 		m_log->log(dht_logger::tracker, "*** new write key***");
 #endif
 	}
 
+	void dht_tracker::update_storage_node_ids()
+	{
+		std::vector<sha1_hash> ids;
+		ids.push_back(m_dht.nid());
+#if TORRENT_USE_IPV6
+		ids.push_back(m_dht6.nid());
+#endif
+		m_storage.update_node_ids(ids);
+	}
+
 /*
 #if defined TORRENT_DEBUG && TORRENT_USE_IOSTREAM
 		std::ofstream st("dht_routing_table_state.txt", std::ios_base::trunc);

src/kademlia/node.cpp

@@ -103,7 +103,7 @@ node::node(udp proto, udp_socket_interface* sock
 	, dht_observer* observer
 	, struct counters& cnt
 	, std::map<std::string, node*> const& nodes
-	, dht_storage_constructor_type storage_constructor)
+	, dht_storage_interface& storage)
 	: m_settings(settings)
 	, m_id(calculate_node_id(nid, observer, proto))
 	, m_table(m_id, proto, 8, settings, observer)
@@ -115,12 +115,10 @@ node::node(udp proto, udp_socket_interface* sock
 	, m_last_self_refresh(min_time())
 	, m_sock(sock)
 	, m_counters(cnt)
-	, m_storage(storage_constructor(m_id, m_settings))
+	, m_storage(storage)
 {
 	m_secret[0] = random();
 	m_secret[1] = random();
-
-	TORRENT_ASSERT(m_storage.get() != NULL);
 }
 
 node::~node() {}
@@ -724,7 +722,7 @@ time_duration node::connection_timeout()
 	if (now - minutes(2) < m_last_tracker_tick) return d;
 	m_last_tracker_tick = now;
 
-	m_storage->tick();
+	m_storage.tick();
 
 	return d;
 }
@@ -750,7 +748,7 @@ void node::status(std::vector<dht_routing_bucket>& table
 // related ones.
 void node::update_stats_counters(counters& c) const
 {
-	const dht_storage_counters& dht_cnt = m_storage->counters();
+	const dht_storage_counters& dht_cnt = m_storage.counters();
 	c.set_value(counters::dht_torrents, dht_cnt.torrents);
 	c.set_value(counters::dht_peers, dht_cnt.peers);
 	c.set_value(counters::dht_immutable_data, dht_cnt.immutable_data);
@@ -770,7 +768,7 @@ void node::status(session_status& s)
 	std::lock_guard<std::mutex> l(m_mutex);
 
 	m_table.status(s);
-	s.dht_torrents = int(m_storage->num_torrents());
+	s.dht_torrents = int(m_storage.num_torrents());
 	s.active_requests.clear();
 	s.dht_total_allocations = m_rpc.num_allocated_observers();
 	for (std::set<traversal_algorithm*>::iterator i = m_running_requests.begin()
@@ -789,7 +787,7 @@ void node::lookup_peers(sha1_hash const& info_hash, entry& reply
 	if (m_observer)
 		m_observer->get_peers(info_hash);
 
-	m_storage->get_peers(info_hash, noseed, scrape, reply);
+	m_storage.get_peers(info_hash, noseed, scrape, reply);
 }
 
 void TORRENT_EXTRA_EXPORT write_nodes_entry(entry& n, nodes_t const& nodes)
@@ -976,7 +974,7 @@ void node::incoming_request(msg const& m, entry& e)
 		std::string name = msg_keys[3] ? msg_keys[3].string_value() : std::string();
 		bool seed = msg_keys[4] && msg_keys[4].int_value();
 
-		m_storage->announce_peer(info_hash, addr, name, seed);
+		m_storage.announce_peer(info_hash, addr, name, seed);
 	}
 	else if (query_len == 3 && memcmp(query, "put", 3) == 0)
 	{
@@ -1059,7 +1057,7 @@ void node::incoming_request(msg const& m, entry& e)
 
 		if (!mutable_put)
 		{
-			m_storage->put_immutable_item(target, buf.first, buf.second, m.addr.address());
+			m_storage.put_immutable_item(target, buf.first, buf.second, m.addr.address());
 		}
 		else
 		{
@@ -1085,9 +1083,9 @@ void node::incoming_request(msg const& m, entry& e)
 			TORRENT_ASSERT(item_sig_len == msg_keys[4].string_length());
 
 			boost::int64_t item_seq;
-			if (!m_storage->get_mutable_item_seq(target, item_seq))
+			if (!m_storage.get_mutable_item_seq(target, item_seq))
 			{
-				m_storage->put_mutable_item(target
+				m_storage.put_mutable_item(target
 					, buf.first, buf.second
 					, sig, seq, pk
 					, salt.first, salt.second
@@ -1114,7 +1112,7 @@ void node::incoming_request(msg const& m, entry& e)
 					return;
 				}
 
-				m_storage->put_mutable_item(target
+				m_storage.put_mutable_item(target
 					, buf.first, buf.second
 					, sig, seq, pk
 					, salt.first, salt.second
@@ -1160,14 +1158,14 @@ void node::incoming_request(msg const& m, entry& e)
 		// so don't bother searching the immutable table
 		if (!msg_keys[0])
 		{
-			if (!m_storage->get_immutable_item(target, reply)) // ok, check for a mutable one
+			if (!m_storage.get_immutable_item(target, reply)) // ok, check for a mutable one
 			{
-				m_storage->get_mutable_item(target, 0, true, reply);
+				m_storage.get_mutable_item(target, 0, true, reply);
 			}
 		}
 		else
 		{
-			m_storage->get_mutable_item(target
+			m_storage.get_mutable_item(target
 				, msg_keys[0].int_value(), false
 				, reply);
 		}

src/kademlia/node_id.cpp

@@ -31,6 +31,7 @@ POSSIBILITY OF SUCH DAMAGE.
 */
 
 #include <algorithm>
+#include <limits>
 
 #include "libtorrent/kademlia/node_id.hpp"
 #include "libtorrent/kademlia/node_entry.hpp"
@@ -70,6 +71,25 @@ int distance_exp(node_id const& n1, node_id const& n2)
 	return (std::max)(159 - distance(n1, n2).count_leading_zeroes(), 0);
 }
 
+int min_distance_exp(node_id const& n1, std::vector<node_id> const& ids)
+{
+	// specialized for cases of 0, 1 and 2 for performance reasons
+	if (ids.size() == 0) return 0;
+	if (ids.size() == 1) return distance_exp(n1, ids[0]);
+	if (ids.size() == 2)
+		return std::min(distance_exp(n1, ids[0]), distance_exp(n1, ids[1]));
+
+	int min = std::numeric_limits<int>::max();
+	for (const auto &node_id : ids)
+	{
+		int d = distance_exp(n1, node_id);
+		if (d < min)
+			min = d;
+	}
+
+	return min;
+}
+
 node_id generate_id_impl(address const& ip_, boost::uint32_t r)
 {
 	boost::uint8_t* ip = 0;

src/session_impl.cpp

@@ -5571,13 +5571,15 @@ namespace aux {
 		// postpone starting the DHT if we're still resolving the DHT router
 		if (m_outstanding_router_lookups > 0) return;
 
+		m_dht_storage = boost::shared_ptr<dht::dht_storage_interface>(
+			m_dht_storage_constructor(m_dht_settings));
 		m_dht = boost::make_shared<dht::dht_tracker>(
 			static_cast<dht_observer*>(this)
 			, boost::ref(m_io_service)
 			, std::bind(&session_impl::send_udp_packet, this, false, _1, _2, _3, _4)
 			, boost::cref(m_dht_settings)
 			, boost::ref(m_stats_counters)
-			, m_dht_storage_constructor
+			, *m_dht_storage
 			, startup_state);
 
 		for (std::vector<udp::endpoint>::iterator i = m_dht_router_nodes.begin()
@@ -5598,11 +5600,18 @@ namespace aux {
 
 	void session_impl::stop_dht()
 	{
-		if (!m_dht) return;
+		if (m_dht)
+		{
 			m_dht->stop();
 			m_dht.reset();
 		}
 
+		if (m_dht_storage)
+		{
+			m_dht_storage.reset();
+		}
+	}
+
 	void session_impl::set_dht_settings(dht_settings const& settings)
 	{
 		m_dht_settings = settings;

test/test_dht.cpp

@@ -508,10 +508,11 @@ void do_test_dht(address(&rand_addr)())
 	mock_socket s;
 	obs observer;
 	counters cnt;
+	boost::scoped_ptr<dht_storage_interface> dht_storage(dht_default_storage_constructor(sett));
 	udp::endpoint source(rand_addr(), 20);
 	std::map<std::string, node*> nodes;
 	dht::node node(source.protocol(), &s, sett
-		, node_id(0), &observer, cnt, nodes);
+		, node_id(0), &observer, cnt, nodes, *dht_storage);
 
 	// DHT should be running on port 48199 now
 	bdecode_node response;
@@ -1425,7 +1426,8 @@ void do_test_dht(address(&rand_addr)())
 	g_sent_packets.clear();
 	do
 	{
-		dht::node node(udp::v4(), &s, sett, (node_id::min)(), &observer, cnt, nodes);
+		boost::scoped_ptr<dht_storage_interface> dht_storage(dht_default_storage_constructor(sett));
+		dht::node node(udp::v4(), &s, sett, (node_id::min)(), &observer, cnt, nodes, *dht_storage);
 
 		udp::endpoint initial_node(address_v4::from_string("4.4.4.4"), 1234);
 		std::vector<udp::endpoint> nodesv;
@@ -1496,8 +1498,9 @@ void do_test_dht(address(&rand_addr)())
 	g_sent_packets.clear();
 	do
 	{
+		boost::scoped_ptr<dht_storage_interface> dht_storage(dht_default_storage_constructor(sett));
 		dht::node_id target = to_hash("1234876923549721020394873245098347598635");
-		dht::node node(udp::v4(), &s, sett, (node_id::min)(), &observer, cnt, nodes);
+		dht::node node(udp::v4(), &s, sett, (node_id::min)(), &observer, cnt, nodes, *dht_storage);
 
 		udp::endpoint initial_node(address_v4::from_string("4.4.4.4"), 1234);
 		node.m_table.add_node(initial_node);
@@ -1592,7 +1595,8 @@ void do_test_dht(address(&rand_addr)())
 	g_sent_packets.clear();
 	do
 	{
-		dht::node node(udp::v4(), &s, sett, (node_id::min)(), &observer, cnt, nodes);
+		boost::scoped_ptr<dht_storage_interface> dht_storage(dht_default_storage_constructor(sett));
+		dht::node node(udp::v4(), &s, sett, (node_id::min)(), &observer, cnt, nodes, *dht_storage);
 
 		udp::endpoint initial_node(address_v4::from_string("4.4.4.4"), 1234);
 		node.m_table.add_node(initial_node);
@@ -1638,7 +1642,8 @@ void do_test_dht(address(&rand_addr)())
 	g_sent_packets.clear();
 	do
 	{
-		dht::node node(udp::v4(), &s, sett, (node_id::min)(), &observer, cnt, nodes);
+		boost::scoped_ptr<dht_storage_interface> dht_storage(dht_default_storage_constructor(sett));
+		dht::node node(udp::v4(), &s, sett, (node_id::min)(), &observer, cnt, nodes, *dht_storage);
 
 		udp::endpoint initial_node(address_v4::from_string("4.4.4.4"), 1234);
 		node.m_table.add_node(initial_node);
@@ -1725,7 +1730,8 @@ void do_test_dht(address(&rand_addr)())
 		// set the branching factor to k to make this a little easier
 		int old_branching = sett.search_branching;
 		sett.search_branching = 8;
-		dht::node node(udp::v4(), &s, sett, (node_id::min)(), &observer, cnt, nodes);
+		boost::scoped_ptr<dht_storage_interface> dht_storage(dht_default_storage_constructor(sett));
+		dht::node node(udp::v4(), &s, sett, (node_id::min)(), &observer, cnt, nodes, *dht_storage);
 		enum { num_test_nodes = 8 };
 		node_entry nodes[num_test_nodes] =
 			{ node_entry(items[0].target, udp::endpoint(address_v4::from_string("1.1.1.1"), 1231))
@@ -1825,7 +1831,8 @@ void do_test_dht(address(&rand_addr)())
 		// set the branching factor to k to make this a little easier
 		int old_branching = sett.search_branching;
 		sett.search_branching = 8;
-		dht::node node(udp::v4(), &s, sett, (node_id::min)(), &observer, cnt, nodes);
+		boost::scoped_ptr<dht_storage_interface> dht_storage(dht_default_storage_constructor(sett));
+		dht::node node(udp::v4(), &s, sett, (node_id::min)(), &observer, cnt, nodes, *dht_storage);
 		enum { num_test_nodes = 8 };
 		node_entry nodes[num_test_nodes] =
 			{ node_entry(items[0].target, udp::endpoint(address_v4::from_string("1.1.1.1"), 1231))
@@ -1927,7 +1934,8 @@ void do_test_dht(address(&rand_addr)())
 		// set the branching factor to k to make this a little easier
 		int old_branching = sett.search_branching;
 		sett.search_branching = 8;
-		dht::node node(udp::v4(), &s, sett, (node_id::min)(), &observer, cnt, nodes);
+		boost::scoped_ptr<dht_storage_interface> dht_storage(dht_default_storage_constructor(sett));
+		dht::node node(udp::v4(), &s, sett, (node_id::min)(), &observer, cnt, nodes, *dht_storage);
 		sha1_hash target = hasher(public_key, item_pk_len).final();
 		enum { num_test_nodes = 9 }; // we need K + 1 nodes to create the failing sequence
 		node_entry nodes[num_test_nodes] =
@@ -2023,8 +2031,9 @@ TORRENT_TEST(dht_dual_stack)
 	obs observer;
 	counters cnt;
 	std::map<std::string, node*> nodes;
-	dht::node node4(udp::v4(), &s, sett, node_id(0), &observer, cnt, nodes);
+	boost::scoped_ptr<dht_storage_interface> dht_storage(dht_default_storage_constructor(sett));
-	dht::node node6(udp::v6(), &s, sett, node_id(0), &observer, cnt, nodes);
+	dht::node node4(udp::v4(), &s, sett, node_id(0), &observer, cnt, nodes, *dht_storage);
+	dht::node node6(udp::v6(), &s, sett, node_id(0), &observer, cnt, nodes, *dht_storage);
 	nodes.insert(std::make_pair("n4", &node4));
 	nodes.insert(std::make_pair("n6", &node6));
 
@@ -2483,7 +2492,8 @@ TORRENT_TEST(read_only_node)
 	counters cnt;
 	std::map<std::string, node*> nodes;
 
-	dht::node node(udp::v4(), &s, sett, node_id(0), &observer, cnt, nodes);
+	boost::scoped_ptr<dht_storage_interface> dht_storage(dht_default_storage_constructor(sett));
+	dht::node node(udp::v4(), &s, sett, node_id(0), &observer, cnt, nodes, *dht_storage);
 	udp::endpoint source(address::from_string("10.0.0.1"), 20);
 	bdecode_node response;
 	msg_args args;
@@ -2572,7 +2582,8 @@ TORRENT_TEST(invalid_error_msg)
 	counters cnt;
 	std::map<std::string, node*> nodes;
 
-	dht::node node(udp::v4(), &s, sett, node_id(0), &observer, cnt, nodes);
+	boost::scoped_ptr<dht_storage_interface> dht_storage(dht_default_storage_constructor(sett));
+	dht::node node(udp::v4(), &s, sett, node_id(0), &observer, cnt, nodes, *dht_storage);
 	udp::endpoint source(address::from_string("10.0.0.1"), 20);
 
 	entry e;
@@ -2612,7 +2623,8 @@ TORRENT_TEST(rpc_invalid_error_msg)
 
 	dht::routing_table table(node_id(), udp::v4(), 8, sett, &observer);
 	dht::rpc_manager rpc(node_id(), sett, table, &s, &observer);
-	dht::node node(udp::v4(), &s, sett, node_id(0), &observer, cnt, nodes);
+	boost::scoped_ptr<dht_storage_interface> dht_storage(dht_default_storage_constructor(sett));
+	dht::node node(udp::v4(), &s, sett, node_id(0), &observer, cnt, nodes, *dht_storage);
 
 	udp::endpoint source(address::from_string("10.0.0.1"), 20);
 
@@ -2692,6 +2704,39 @@ TORRENT_TEST(node_id_bucket_distribution)
 	}
 }
 
+TORRENT_TEST(node_id_min_distance_exp)
+{
+	node_id n1 = to_hash("0000000000000000000000000000000000000002");
+	node_id n2 = to_hash("0000000000000000000000000000000000000004");
+	node_id n3 = to_hash("0000000000000000000000000000000000000008");
+
+	std::vector<node_id> ids;
+
+	TEST_EQUAL(min_distance_exp(sha1_hash::min(), ids), 0);
+
+	ids.push_back(n1);
+
+	TEST_EQUAL(min_distance_exp(sha1_hash::min(), ids), 1);
+
+	ids.push_back(n1);
+	ids.push_back(n2);
+
+	TEST_EQUAL(min_distance_exp(sha1_hash::min(), ids), 1);
+
+	ids.push_back(n1);
+	ids.push_back(n2);
+	ids.push_back(n3);
+
+	TEST_EQUAL(min_distance_exp(sha1_hash::min(), ids), 1);
+
+	ids.clear();
+	ids.push_back(n3);
+	ids.push_back(n2);
+	ids.push_back(n2);
+
+	TEST_EQUAL(min_distance_exp(sha1_hash::min(), ids), 2);
+}
+
 TORRENT_TEST(dht_verify_node_address)
 {
 	obs observer;

test/test_dht_storage.cpp

@@ -76,11 +76,10 @@ namespace
 
 	bool g_storage_constructor_invoked = false;
 
-	dht_storage_interface* dht_custom_storage_constructor(sha1_hash const& id
+	dht_storage_interface* dht_custom_storage_constructor(dht_settings const& settings)
-		, dht_settings const& settings)
 	{
 		g_storage_constructor_invoked = true;
-		return dht_default_storage_constructor(id, settings);
+		return dht_default_storage_constructor(settings);
 	}
 }
 
@@ -92,7 +91,7 @@ const sha1_hash n4 = to_hash("5fbfbff10c5d6a4ec8a88e4c6ab4c28b95eee404");
 TORRENT_TEST(announce_peer)
 {
 	dht_settings sett = test_settings();
-	boost::scoped_ptr<dht_storage_interface> s(dht_default_storage_constructor(node_id(0), sett));
+	boost::scoped_ptr<dht_storage_interface> s(dht_default_storage_constructor(sett));
 	TEST_CHECK(s.get() != NULL);
 
 	entry peers;
@@ -121,7 +120,7 @@ TORRENT_TEST(announce_peer)
 TORRENT_TEST(put_immutable_item)
 {
 	dht_settings sett = test_settings();
-	boost::scoped_ptr<dht_storage_interface> s(dht_default_storage_constructor(node_id(0), sett));
+	boost::scoped_ptr<dht_storage_interface> s(dht_default_storage_constructor(sett));
 	TEST_CHECK(s.get() != NULL);
 
 	entry item;
@@ -151,7 +150,7 @@ TORRENT_TEST(put_immutable_item)
 TORRENT_TEST(counters)
 {
 	dht_settings sett = test_settings();
-	boost::scoped_ptr<dht_storage_interface> s(dht_default_storage_constructor(node_id(0), sett));
+	boost::scoped_ptr<dht_storage_interface> s(dht_default_storage_constructor(sett));
 
 	TEST_CHECK(s.get() != NULL);
 
@@ -240,7 +239,7 @@ TORRENT_TEST(peer_limit)
 {
 	dht_settings sett = test_settings();
 	sett.max_peers = 42;
-	boost::scoped_ptr<dht_storage_interface> s(dht_default_storage_constructor(node_id(0), sett));
+	boost::scoped_ptr<dht_storage_interface> s(dht_default_storage_constructor(sett));
 	TEST_CHECK(s.get() != NULL);
 
 	for (int i = 0; i < 200; ++i)
@@ -258,7 +257,7 @@ TORRENT_TEST(torrent_limit)
 {
 	dht_settings sett = test_settings();
 	sett.max_torrents = 42;
-	boost::scoped_ptr<dht_storage_interface> s(dht_default_storage_constructor(node_id(0), sett));
+	boost::scoped_ptr<dht_storage_interface> s(dht_default_storage_constructor(sett));
 	TEST_CHECK(s.get() != NULL);
 
 	for (int i = 0; i < 200; ++i)
@@ -276,7 +275,7 @@ TORRENT_TEST(immutable_item_limit)
 {
 	dht_settings sett = test_settings();
 	sett.max_dht_items = 42;
-	boost::scoped_ptr<dht_storage_interface> s(dht_default_storage_constructor(node_id(0), sett));
+	boost::scoped_ptr<dht_storage_interface> s(dht_default_storage_constructor(sett));
 	TEST_CHECK(s.get() != NULL);
 
 	for (int i = 0; i < 200; ++i)
@@ -293,7 +292,7 @@ TORRENT_TEST(mutable_item_limit)
 {
 	dht_settings sett = test_settings();
 	sett.max_dht_items = 42;
-	boost::scoped_ptr<dht_storage_interface> s(dht_default_storage_constructor(node_id(0), sett));
+	boost::scoped_ptr<dht_storage_interface> s(dht_default_storage_constructor(sett));
 	TEST_CHECK(s.get() != NULL);
 
 	char public_key[item_pk_len];
@@ -308,5 +307,55 @@ TORRENT_TEST(mutable_item_limit)
 	TEST_EQUAL(cnt.mutable_data, 42);
 }
 
+TORRENT_TEST(update_node_ids)
+{
+	dht_settings sett = test_settings();
+	boost::scoped_ptr<dht_storage_interface> s(dht_default_storage_constructor(sett));
+	TEST_CHECK(s.get() != NULL);
+
+	node_id n1 = to_hash("0000000000000000000000000000000000000200");
+	node_id n2 = to_hash("0000000000000000000000000000000000000400");
+	node_id n3 = to_hash("0000000000000000000000000000000000000800");
+
+	std::vector<node_id> node_ids;
+	node_ids.push_back(n1);
+	node_ids.push_back(n2);
+	node_ids.push_back(n3);
+	s->update_node_ids(node_ids);
+
+	entry item;
+	dht_storage_counters cnt;
+	bool r;
+
+	sha1_hash h1 = to_hash("0000000000000000000000000000000000010200");
+	sha1_hash h2 = to_hash("0000000000000000000000000000000100000400");
+	sha1_hash h3 = to_hash("0000000000000000000000010000000000000800");
+
+	TEST_EQUAL(min_distance_exp(h1, node_ids), 16);
+	TEST_EQUAL(min_distance_exp(h2, node_ids), 32);
+	TEST_EQUAL(min_distance_exp(h3, node_ids), 64);
+
+	// all items will have one announcer, all calculations
+	// for item erase will be reduced to the distance
+	s->put_immutable_item(h1, "123", 3, address::from_string("124.31.75.21"));
+	cnt = s->counters();
+	TEST_EQUAL(cnt.immutable_data, 1);
+	s->put_immutable_item(h2, "123", 3, address::from_string("124.31.75.21"));
+	cnt = s->counters();
+	TEST_EQUAL(cnt.immutable_data, 2);
+	// at this point, the least important (h2) will removed
+	// to make room for h3
+	s->put_immutable_item(h3, "123", 3, address::from_string("124.31.75.21"));
+	cnt = s->counters();
+	TEST_EQUAL(cnt.immutable_data, 2);
+
+	r = s->get_immutable_item(h1, item);
+	TEST_CHECK(r);
+	r = s->get_immutable_item(h2, item);
+	TEST_CHECK(!r);
+	r = s->get_immutable_item(h3, item);
+	TEST_CHECK(r);
+}
+
 #endif