peer list optimization and introduced hard limit on peer list size

docs/manual.rst

@@ -2760,6 +2760,8 @@ that will be sent to the tracker. The user-agent is a good way to identify your
 
 		int auto_scrape_interval;
 		int auto_scrape_min_interval;
+
+		int max_peerlist_size;
 	};
 
 ``user_agent`` this is the client identification to the tracker.
@@ -2996,6 +2998,12 @@ automatic scrape (regardless of torrent). In case there are a large number
 of paused auto managed torrents, this puts a limit on how often a scrape
 request is sent.
 
+``max_peerlist_size`` is the maximum number of peers in the list of
+known peers. These peers are not necessarily connected, so this number
+should be much greater than the maximum number of connected peers.
+Peers are evicted from the cache when the list grows passed 90% of
+this limit, and once the size hits the limit, peers are no longer
+added to the list.
 
 pe_settings
 ===========

include/libtorrent/session_settings.hpp

@@ -135,6 +135,7 @@ namespace libtorrent
 			, close_redundant_connections(true)
 			, auto_scrape_interval(1800)
 			, auto_scrape_min_interval(300)
+			, max_peerlist_size(8000)
 		{}
 
 		// this is the user agent that will be sent to the tracker
@@ -402,6 +403,11 @@ namespace libtorrent
 		// the minimum number of seconds between any
 		// automatic scrape (regardless of torrent)
 		int auto_scrape_min_interval;
+
+		// the max number of peers in the peer list
+		// per torrent. This is the peers we know
+		// about, not necessarily connected to.
+		int max_peerlist_size;
 	};
 
 #ifndef TORRENT_DISABLE_DHT

src/policy.cpp

@@ -150,18 +150,7 @@ namespace
 		tcp::endpoint const& m_ep;
 	};
 
-	struct match_peer_id
+#ifndef NDEBUG
-	{
-		match_peer_id(peer_id const& id_)
-			: m_id(id_)
-		{}
-
-		bool operator()(std::pair<const address, policy::peer> const& p) const
-		{ return p.second.connection && p.second.connection->pid() == m_id; }
-
-		peer_id const& m_id;
-	};
-
 	struct match_peer_connection
 	{
 		match_peer_connection(peer_connection const& c)
@@ -177,7 +166,7 @@ namespace
 
 		peer_connection const& m_conn;
 	};
-
+#endif
 
 }
 
@@ -445,21 +434,56 @@ namespace libtorrent
 
 		if (m_round_robin == m_peers.end()) m_round_robin = m_peers.begin();
 
+#ifndef TORRENT_DISABLE_DHT
+		bool pinged = false;
+#endif
+
 		for (int iterations = (std::min)(int(m_peers.size()), 300);
-			iterations > 0; ++m_round_robin, --iterations)
+			iterations > 0; --iterations)
 		{
 			if (m_round_robin == m_peers.end()) m_round_robin = m_peers.begin();
 
-			if (!is_connect_candidate(m_round_robin->second, finished)) continue;
+			peer& pe = m_round_robin->second;
+			iterator current = m_round_robin;
+
+#ifndef TORRENT_DISABLE_DHT
+			// try to send a DHT ping to this peer
+			// as well, to figure out if it supports
+			// DHT (uTorrent and BitComet doesn't
+			// advertise support)
+			if (!pinged && !pe.added_to_dht)
+			{
+				udp::endpoint node(pe.ip.address(), pe.ip.port());
+				m_torrent->session().add_dht_node(node);
+				pe.added_to_dht = true;
+				pinged = true;
+			}
+#endif
+			// this timeout has to be customizable!
+			// don't remove banned peers, they should
+			// remain banned
+			if (pe.connection == 0
+				&& pe.connected != min_time()
+				&& !pe.banned
+				&& (now - pe.connected > minutes(120)
+					|| m_peers.size() >= m_torrent->settings().max_peerlist_size) * 0.9)
+			{
+				erase_peer(m_round_robin++);
+				continue;
+			}
+			++m_round_robin;
+
+
+			if (!is_connect_candidate(pe, finished)) continue;
 
 			if (candidate != m_peers.end()
-				&& !compare_peer(candidate->second, m_round_robin->second, external_ip)) continue;
+				&& !compare_peer(candidate->second, pe, external_ip)) continue;
 
-			if (now - m_round_robin->second.connected <
+			if (now - pe.connected <
-				seconds(m_round_robin->second.failcount * min_reconnect_time))
+				seconds(pe.failcount * min_reconnect_time))
 				continue;
 
-			candidate = m_round_robin;
+			candidate = current;
 		}
 		
 #if defined TORRENT_LOGGING || defined TORRENT_VERBOSE_LOGGING
@@ -484,45 +508,6 @@ namespace libtorrent
 
 		if (m_torrent->is_paused()) return;
 
-#ifndef TORRENT_DISABLE_DHT
-		bool pinged = false;
-#endif
-
-		ptime now = time_now();
-		// remove old disconnected peers from the list
-		for (iterator i = m_peers.begin(); i != m_peers.end();)
-		{
-			peer& pe = i->second;
-
-#ifndef TORRENT_DISABLE_DHT
-			// try to send a DHT ping to this peer
-			// as well, to figure out if it supports
-			// DHT (uTorrent and BitComet doesn't
-			// advertise support)
-			if (!pinged && !pe.added_to_dht)
-			{
-				udp::endpoint node(pe.ip.address(), pe.ip.port());
-				m_torrent->session().add_dht_node(node);
-				pe.added_to_dht = true;
-				pinged = true;
-			}
-#endif
-			// this timeout has to be customizable!
-			// don't remove banned peers, they should
-			// remain banned
-			if (pe.connection == 0
-				&& pe.connected != min_time()
-				&& !pe.banned
-				&& now - pe.connected > minutes(120))
-			{
-				erase_peer(i++);
-			}
-			else
-			{
-				++i;
-			}
-		}
-
 		// ------------------------
 		// upload shift
 		// ------------------------
@@ -658,6 +643,12 @@ namespace libtorrent
 			error_code ec;
 			TORRENT_ASSERT(c.remote() == c.get_socket()->remote_endpoint(ec) || ec);
 
+			if (m_peers.size() >= m_torrent->settings().max_peerlist_size)
+			{
+				c.disconnect("peer list size exceeded, refusing incoming connection");
+				return false;
+			}
+
 			peer p(c.remote(), peer::not_connectable, 0);
 			i = m_peers.insert(std::make_pair(c.remote().address(), p));
 #ifndef TORRENT_DISABLE_GEO_IP
@@ -781,6 +772,9 @@ namespace libtorrent
 				return 0;
 			}
 
+			if (m_peers.size() >= m_torrent->settings().max_peerlist_size)
+				return 0;
+
 			// we don't have any info about this peer.
 			// add a new entry
 			i = m_peers.insert(std::make_pair(remote.address()