add limit for number of web seed connections (#1817)

add limit for number of web seed connections

ChangeLog

@@ -1,3 +1,4 @@
+	* add limit for number of web seed connections
 	* added support for retrieval of DHT live nodes
 	* complete UNC path support
 	* add packets pool allocator

include/libtorrent/settings_pack.hpp

@@ -1625,6 +1625,10 @@ namespace libtorrent
 			// systems.
 			close_file_interval,
 
+			// the max number of web seeds to have connected per torrent at any
+			// given time.
+			max_web_seed_connections,
+
 			max_int_setting_internal
 		};
 

include/libtorrent/torrent.hpp

@@ -183,6 +183,31 @@ namespace libtorrent
 		// if this bitfield is non-empty, it represents the files this web server
 		// has.
 		typed_bitfield<file_index_t> have_files;
+#if defined __GNUC__ && defined _GLIBCXX_DEBUG
+		// this works around a bug in libstdc++'s checked iterators
+		// http://stackoverflow.com/questions/22915325/avoiding-self-assignment-in-stdshuffle
+		web_seed_t& operator=(web_seed_t&& rhs)
+		{
+			if (&rhs == this) return *this;
+
+			web_seed_entry::operator=(std::move(rhs));
+			retry = std::move(rhs.retry);
+			endpoints = std::move(rhs.endpoints);
+			peer_info = std::move(rhs.peer_info);
+			supports_keepalive = std::move(rhs.supports_keepalive);
+			resolving = std::move(rhs.resolving);
+			removed = std::move(rhs.removed);
+			ephemeral = std::move(rhs.ephemeral);
+			restart_request = std::move(rhs.restart_request);
+			restart_piece = std::move(rhs.restart_piece);
+			redirects = std::move(rhs.redirects);
+			have_files = std::move(rhs.have_files);
+			return *this;
+		}
+
+		web_seed_t& operator=(web_seed_t const&) = default;
+		web_seed_t(web_seed_t const&) = default;
+#endif
 	};
 
 	struct TORRENT_EXTRA_EXPORT torrent_hot_members

include/libtorrent/torrent_peer.hpp

@@ -201,6 +201,7 @@ namespace libtorrent
 	{
 		ipv4_peer(tcp::endpoint const& ip, bool connectable, int src);
 		ipv4_peer(ipv4_peer const& p);
+		ipv4_peer& operator=(ipv4_peer const& p);
 
 		address_v4 addr;
 	};
@@ -209,7 +210,7 @@ namespace libtorrent
 	struct TORRENT_EXTRA_EXPORT i2p_peer : torrent_peer
 	{
 		i2p_peer(char const* destination, bool connectable, int src);
-		i2p_peer(const i2p_peer&);
+		i2p_peer(i2p_peer const&);
 		~i2p_peer();
 		i2p_peer& operator=(i2p_peer const&);
 
@@ -221,6 +222,7 @@ namespace libtorrent
 	struct TORRENT_EXTRA_EXPORT ipv6_peer : torrent_peer
 	{
 		ipv6_peer(tcp::endpoint const& ip, bool connectable, int src);
+		ipv6_peer(ipv6_peer const& p);
 
 		const address_v6::bytes_type addr;
 	};
@@ -229,7 +231,7 @@ namespace libtorrent
 	struct peer_address_compare
 	{
 		bool operator()(
-				torrent_peer const* lhs, libtorrent::address const& rhs) const
+			torrent_peer const* lhs, libtorrent::address const& rhs) const
 		{
 			return lhs->address() < rhs;
 		}

simulation/test_web_seed.cpp

@@ -45,6 +45,7 @@ POSSIBILITY OF SUCH DAMAGE.
 #include "make_proxy_settings.hpp"
 #include "simulator/utils.hpp"
 #include <iostream>
+#include <numeric>
 
 using namespace sim;
 using namespace libtorrent;
@@ -105,7 +106,8 @@ add_torrent_params create_torrent(file_storage& fs, bool const pad_files = false
 template <typename Setup, typename HandleAlerts, typename Test>
 void run_test(Setup const& setup
 	, HandleAlerts const& on_alert
-	, Test const& test)
+	, Test const& test
+	, lt::seconds const timeout = lt::seconds{100})
 {
 	// setup the simulation
 	sim::default_config network_cfg;
@@ -127,7 +129,7 @@ void run_test(Setup const& setup
 
 	// set up a timer to fire later, to verify everything we expected to happen
 	// happened
-	sim::timer t(sim, lt::seconds(100), [&](boost::system::error_code const& ec)
+	sim::timer t(sim, timeout, [&](boost::system::error_code const& ec)
 	{
 		std::printf("shutting down\n");
 		// shut down
@@ -572,3 +574,65 @@ TORRENT_TEST(no_close_redudant_webseed)
 
 	TEST_CHECK(expected);
 }
+
+// make sure the max_web_seed_connections limit is honored
+TORRENT_TEST(web_seed_connection_limit)
+{
+	using namespace libtorrent;
+
+	file_storage fs;
+	fs.add_file("file1", 1);
+	lt::add_torrent_params params = ::create_torrent(fs);
+	params.url_seeds.push_back("http://2.2.2.1:8080/");
+	params.url_seeds.push_back("http://2.2.2.2:8080/");
+	params.url_seeds.push_back("http://2.2.2.3:8080/");
+	params.url_seeds.push_back("http://2.2.2.4:8080/");
+
+	std::array<int, 4> expected = {};
+	run_test(
+		[&params](lt::session& ses)
+		{
+			lt::settings_pack pack;
+			pack.set_int(settings_pack::max_web_seed_connections, 2);
+			ses.apply_settings(pack);
+			ses.async_add_torrent(params);
+		},
+		[](lt::session& ses, lt::alert const* alert) {},
+		[&expected](sim::simulation& sim, lt::session& ses)
+		{
+			using ios = sim::asio::io_service;
+			ios web_server1{sim, address_v4::from_string("2.2.2.1")};
+			ios web_server2{sim, address_v4::from_string("2.2.2.2")};
+			ios web_server3{sim, address_v4::from_string("2.2.2.3")};
+			ios web_server4{sim, address_v4::from_string("2.2.2.4")};
+
+			// listen on port 8080
+			using ws = sim::http_server;
+			ws http1{web_server1, 8080};
+			ws http2{web_server2, 8080};
+			ws http3{web_server3, 8080};
+			ws http4{web_server4, 8080};
+
+			auto const handler = [&expected](std::string method, std::string req
+				, std::map<std::string, std::string>&, int idx)
+			{
+				++expected[idx];
+				// deliberately avoid sending the content, to cause a hang
+				return sim::send_response(206, "Partial Content", 1);
+			};
+
+			using namespace std::placeholders;
+			http1.register_handler("/file1", std::bind(handler, _1, _2, _3, 0));
+			http2.register_handler("/file1", std::bind(handler, _1, _2, _3, 1));
+			http3.register_handler("/file1", std::bind(handler, _1, _2, _3, 2));
+			http4.register_handler("/file1", std::bind(handler, _1, _2, _3, 3));
+
+			sim.run();
+		},
+		lt::seconds(15)
+	);
+
+	// make sure we only connected to 2 of the web seeds, since that's the limit
+	TEST_CHECK(std::accumulate(expected.begin(), expected.end(), 0) == 2);
+}
+

src/settings_pack.cpp

@@ -334,6 +334,7 @@ namespace libtorrent
 		SET(urlseed_max_request_bytes, 16 * 1024 * 1024, 0),
 		SET(web_seed_name_lookup_retry, 1800, nullptr),
 		SET(close_file_interval, CLOSE_FILE_INTERVAL, &session_impl::update_close_file_interval),
+		SET(max_web_seed_connections, 3, nullptr),
 	}});
 
 #undef SET

src/torrent.cpp

@@ -250,10 +250,11 @@ namespace libtorrent
 
 		// if override web seed flag is set, don't load any web seeds from the
 		// torrent file.
+		std::vector<web_seed_t> ws;
 		if ((p.flags & add_torrent_params::flag_override_web_seeds) == 0)
 		{
-			std::vector<web_seed_entry> const& web_seeds = m_torrent_file->web_seeds();
-			m_web_seeds.insert(m_web_seeds.end(), web_seeds.begin(), web_seeds.end());
+			for (auto const& e : m_torrent_file->web_seeds())
+				ws.emplace_back(e);
 		}
 
 		// add web seeds from add_torrent_params
@@ -262,17 +263,18 @@ namespace libtorrent
 
 		for (auto const& u : p.url_seeds)
 		{
-			m_web_seeds.push_back(web_seed_t(u, web_seed_entry::url_seed));
+			ws.emplace_back(web_seed_t(u, web_seed_entry::url_seed));
 
 			// correct URLs to end with a "/" for multi-file torrents
-			std::string& url = m_web_seeds.back().url;
+			std::string& url = ws.back().url;
 			if (multi_file && url[url.size()-1] != '/') url += '/';
 		}
 
 		for (auto const& e : p.http_seeds)
-		{
-			m_web_seeds.push_back(web_seed_t(e, web_seed_entry::http_seed));
-		}
+			ws.push_back(web_seed_t(e, web_seed_entry::http_seed));
+
+		aux::random_shuffle(ws.begin(), ws.end());
+		for (auto& w : ws) m_web_seeds.emplace_back(std::move(w));
 
 		// --- TRACKERS ---
 
@@ -470,7 +472,9 @@ namespace libtorrent
 
 		// add the web seeds from the .torrent file
 		std::vector<web_seed_entry> const& web_seeds = m_torrent_file->web_seeds();
-		m_web_seeds.insert(m_web_seeds.end(), web_seeds.begin(), web_seeds.end());
+		std::vector<web_seed_t> ws(web_seeds.begin(), web_seeds.end());
+		aux::random_shuffle(ws.begin(), ws.end());
+		for (auto& w : ws) m_web_seeds.push_back(std::move(w));
 
 #if !defined(TORRENT_DISABLE_ENCRYPTION) && !defined(TORRENT_DISABLE_EXTENSIONS)
 		static char const req2[4] = {'r', 'e', 'q', '2'};
@@ -8897,28 +8901,45 @@ namespace libtorrent
 	}
 	catch (...) { handle_exception(); }
 
+	namespace {
+		int zero_or(int const val, int const def_val)
+		{ return (val <= 0) ? def_val : val; }
+	}
+
 	void torrent::maybe_connect_web_seeds()
 	{
 		if (m_abort) return;
 
 		// if we have everything we want we don't need to connect to any web-seed
-		if (!is_finished() && !m_web_seeds.empty() && m_files_checked
-			&& num_peers() < int(m_max_connections)
-			&& m_ses.num_connections() < settings().get_int(settings_pack::connections_limit))
+		if (m_web_seeds.empty()
+			|| is_finished()
+			|| !m_files_checked
+			|| num_peers() >= int(m_max_connections)
+			|| m_ses.num_connections() >= settings().get_int(settings_pack::connections_limit))
 		{
-			// keep trying web-seeds if there are any
-			// first find out which web seeds we are connected to
-			for (std::list<web_seed_t>::iterator i = m_web_seeds.begin();
-				i != m_web_seeds.end();)
-			{
-				std::list<web_seed_t>::iterator w = i++;
-				if (w->peer_info.connection) continue;
-				if (w->retry > aux::time_now()) continue;
-				if (w->resolving) continue;
-				if (w->removed) continue;
+			return;
+		}
 
-				connect_to_url_seed(w);
-			}
+		// when set to unlimited, use 100 as the limit
+		int limit = zero_or(settings().get_int(settings_pack::max_web_seed_connections)
+			, 100);
+
+		auto const now = aux::time_now();
+
+		// keep trying web-seeds if there are any
+		// first find out which web seeds we are connected to
+		for (std::list<web_seed_t>::iterator i = m_web_seeds.begin();
+			i != m_web_seeds.end() && limit > 0;)
+		{
+			std::list<web_seed_t>::iterator w = i++;
+			if (w->removed || w->retry > now)
+				continue;
+
+			--limit;
+			if (w->peer_info.connection || w->resolving)
+				continue;
+
+			connect_to_url_seed(w);
 		}
 	}
 

src/torrent_peer.cpp

@@ -241,6 +241,7 @@ namespace libtorrent
 	}
 
 	ipv4_peer::ipv4_peer(ipv4_peer const&) = default;
+	ipv4_peer& ipv4_peer::operator=(ipv4_peer const& p) = default;
 
 #if TORRENT_USE_I2P
 	i2p_peer::i2p_peer(char const* dest, bool connectable, int src)
@@ -282,6 +283,8 @@ namespace libtorrent
 #endif
 	}
 
+	ipv6_peer::ipv6_peer(ipv6_peer const&) = default;
+
 #endif // TORRENT_USE_IPV6
 
 #if TORRENT_USE_I2P