merged RC_1_1 into master

ChangeLog

@@ -46,6 +46,9 @@
 	* resume data no longer has timestamps of files
 	* require C++11 to build libtorrent
 
+	* fix socks5 support for UDP
+	* add setting urlseed_max_request_bytes to handle large web seed requests
+	* fix python build with CC/CXX environment
 	* add trackers from add_torrent_params/magnet links to separate tiers
 	* fix resumedata check issue with files with priority 0
 	* deprecated mmap_cache feature

bindings/python/setup.py

@@ -114,7 +114,10 @@ else:
 
 		# for some reason distutils uses the CC environment variable to determine
 		# the compiler to use for C++
-		os.environ["CC"] = os.environ['CXX']
+		if 'CXX' in os.environ:
+			os.environ['CC'] = os.environ['CXX']
+		if 'CXXFLAGS' in os.environ:
+			os.environ['CFLAGS'] = os.environ['CXXFLAGS']
 
 		ext = [Extension('libtorrent',
 			sources = source_list,

bindings/python/src/session.cpp

@@ -357,7 +357,7 @@ namespace
           d["piece"] = i->piece;
           d["last_use"] = total_milliseconds(now - i->last_use) / 1000.f;
           d["next_to_hash"] = i->next_to_hash;
-          d["kind"] = i->kind;
+          d["kind"] = static_cast<int>(i->kind);
           pieces.append(d);
        }
        return pieces;

bindings/python/test.py

@@ -29,6 +29,9 @@ class test_torrent_handle(unittest.TestCase):
 		ti = lt.torrent_info('url_seed_multi.torrent');
 		h = ses.add_torrent({'ti': ti, 'save_path': os.getcwd()})
 
+		self.assertEqual(h.file_priorities(), [4,4])
+		self.assertEqual(h.piece_priorities(), [4])
+
 		h.prioritize_files([0,1])
 		self.assertEqual(h.file_priorities(), [0,1])
 

include/libtorrent/peer_connection.hpp

@@ -334,7 +334,7 @@ namespace libtorrent
 		int picker_options() const;
 
 		void prefer_contiguous_blocks(int num)
-		{ m_prefer_contiguous_blocks = std::uint8_t((std::min)(num, 255)); }
+		{ m_prefer_contiguous_blocks = num; }
 
 		bool request_large_blocks() const
 		{ return m_request_large_blocks; }
@@ -1066,7 +1066,7 @@ namespace libtorrent
 		// if it is 0, the download rate limit setting
 		// will be used to determine if whole pieces
 		// are preferred.
-		std::uint8_t m_prefer_contiguous_blocks = 0;
+		int m_prefer_contiguous_blocks = 0;
 
 		// this is the number of times this peer has had
 		// a request rejected because of a disk I/O failure.

include/libtorrent/settings_pack.hpp

@@ -780,6 +780,21 @@ namespace libtorrent
 			// low number, like 5
 			urlseed_pipeline_size,
 
+			// The maximum request range of an url seed in bytes. This value
+			// defines the largest possible sequential web seed request. Default
+			// is 16 * 1024 * 1024. Lower values are possible but will be ignored
+			// if they are lower then piece size.
+			// This value should be related to your download speed to prevent
+			// libtorrent from creating too many expensive http requests per
+			// second. You can select a value as high as you want but keep in mind
+			// that libtorrent can't create parallel requests if the first request
+			// did already select the whole file.
+			// If you combine bittorrent seeds with web seeds and pick strategies
+			// like rarest first you may find your web seed requests split into
+			// smaller parts because we don't download already picked pieces
+			// twice.
+			urlseed_max_request_bytes,
+
 			// time to wait until a new retry of a web seed takes place
 			urlseed_wait_retry,
 

include/libtorrent/torrent_handle.hpp

@@ -961,8 +961,6 @@ namespace libtorrent
 		// will return the resume data in an alert
 		TORRENT_DEPRECATED
 		entry write_resume_data() const;
-		// ================ end deprecation ============
-#endif
 
 		// ``use_interface()`` sets the network interface this torrent will use
 		// when it opens outgoing connections. By default, it uses the same
@@ -971,7 +969,10 @@ namespace libtorrent
 		// IPv4 or IPv6 address). When specifying multiple interfaces, the
 		// torrent will round-robin which interface to use for each outgoing
 		// connection. This is useful for clients that are multi-homed.
+		TORRENT_DEPRECATED
 		void use_interface(const char* net_interface) const;
+		// ================ end deprecation ============
+#endif
 
 		// Fills the specified ``std::vector<int>`` with the availability for
 		// each piece in this torrent. libtorrent does not keep track of

m4/ax_check_openssl.m4

@@ -106,7 +106,7 @@ AC_DEFUN([AX_CHECK_OPENSSL], [
     LIBS="$OPENSSL_LIBS $LIBS"
     CPPFLAGS="$OPENSSL_INCLUDES $CPPFLAGS"
     AC_LINK_IFELSE(
-        AC_LANG_PROGRAM([#include <openssl/ssl.h>], [SSL_new(NULL)]),
+        [AC_LANG_PROGRAM([#include <openssl/ssl.h>], [SSL_new(NULL)])],
         [
             AC_MSG_RESULT([yes])
             $1

simulation/fake_peer.hpp

@@ -55,12 +55,6 @@ struct fake_peer
 {
 	fake_peer(simulation& sim, char const* ip)
 		: m_ios(sim, asio::ip::address::from_string(ip))
-		, m_acceptor(m_ios)
-		, m_socket(m_ios)
-		, m_info_hash(0)
-		, m_accepted(false)
-		, m_connected(false)
-		, m_disconnected(false)
 	{
 		boost::system::error_code ec;
 		m_acceptor.open(asio::ip::tcp::v4(), ec);
@@ -249,19 +243,19 @@ private:
 	char m_out_buffer[300];
 
 	asio::io_service m_ios;
-	asio::ip::tcp::acceptor m_acceptor;
-	asio::ip::tcp::socket m_socket;
+	asio::ip::tcp::acceptor m_acceptor{m_ios};
+	asio::ip::tcp::socket m_socket{m_ios};
 	lt::sha1_hash m_info_hash;
 
 	// set to true if this peer received an incoming connection
 	// if this is an outgoing connection, this will always be false
-	bool m_accepted;
+	bool m_accepted = false;
 
 	// set to true if this peer completed a bittorrent handshake
-	bool m_connected;
+	bool m_connected = false;
 
 	// set to true if this peer has been disconnected by the other end
-	bool m_disconnected;
+	bool m_disconnected = false;
 
 	std::vector<char> m_send_buffer;
 };
@@ -283,12 +277,12 @@ inline void add_fake_peers(lt::torrent_handle& h, int const n = 5)
 	}
 }
 
-struct fake_node
+struct udp_server
 {
-	fake_node(simulation& sim, char const* ip, int port = 6881)
+	udp_server(simulation& sim, char const* ip, int port
+		, std::function<std::vector<char>(char const*, int)> handler)
 		: m_ios(sim, asio::ip::address::from_string(ip))
-		, m_socket(m_ios)
-		, m_tripped(false)
+		, m_handler(handler)
 	{
 		boost::system::error_code ec;
 		m_socket.open(asio::ip::udp::v4(), ec);
@@ -296,31 +290,73 @@ struct fake_node
 		m_socket.bind(asio::ip::udp::endpoint(asio::ip::address_v4::any(), port), ec);
 		TEST_CHECK(!ec);
 
-		std::printf("fake_node::async_read_some\n");
-		m_socket.async_receive(boost::asio::buffer(m_in_buffer)
-			, [&] (boost::system::error_code const& ec, size_t bytes_transferred)
-		{
-			std::printf("fake_node::async_read_some callback. ec: %s transferred: %d\n"
-				, ec.message().c_str(), int(bytes_transferred));
-			if (ec) return;
+		m_socket.io_control(lt::udp::socket::non_blocking_io(true));
 
+		std::printf("udp_server::async_read_some\n");
+		using namespace std::placeholders;
+		m_socket.async_receive_from(boost::asio::buffer(m_in_buffer)
+			, m_from, 0, std::bind(&udp_server::on_read, this, _1, _2));
+	}
+
+	void close() { m_socket.close(); }
+
+private:
+
+	void on_read(boost::system::error_code const& ec, size_t bytes_transferred)
+	{
+		std::printf("udp_server::async_read_some callback. ec: %s transferred: %d\n"
+			, ec.message().c_str(), int(bytes_transferred));
+		if (ec) return;
+
+		std::vector<char> send_buffer = m_handler(m_in_buffer.data(), int(bytes_transferred));
+
+		if (!send_buffer.empty())
+		{
+			lt::error_code err;
+			m_socket.send_to(boost::asio::buffer(send_buffer), m_from, 0, err);
+			if (err)
+			{
+				std::printf("send_to FAILED: %s\n", err.message().c_str());
+			}
+			else
+			{
+				std::printf("udp_server responding with %d bytes\n"
+					, int(send_buffer.size()));
+			}
+		}
+
+		std::printf("udp_server::async_read_some\n");
+		using namespace std::placeholders;
+		m_socket.async_receive_from(boost::asio::buffer(m_in_buffer)
+			, m_from, 0, std::bind(&udp_server::on_read, this, _1, _2));
+	}
+
+	std::array<char, 1500> m_in_buffer;
+
+	asio::io_service m_ios;
+	asio::ip::udp::socket m_socket{m_ios};
+	asio::ip::udp::endpoint m_from;
+
+	std::function<std::vector<char>(char const*, int)> m_handler;
+};
+
+struct fake_node : udp_server
+{
+	fake_node(simulation& sim, char const* ip, int port = 6881)
+		: udp_server(sim, ip, port, [&](char const* incoming, int size)
+		{
 			lt::bdecode_node n;
 			boost::system::error_code err;
-			int const ret = bdecode(m_in_buffer.data(), m_in_buffer.data() + bytes_transferred
-				, n, err, nullptr, 10, 200);
+			int const ret = bdecode(incoming, incoming + size, n, err, nullptr, 10, 200);
 			TEST_EQUAL(ret, 0);
 
-			m_incoming_packets.emplace_back(m_in_buffer.data(), m_in_buffer.data() + bytes_transferred);
+			m_incoming_packets.emplace_back(incoming, incoming + size);
 
 			// TODO: ideally we would validate the DHT message
 			m_tripped = true;
-		});
-	}
-
-	void close()
-	{
-		m_socket.close();
-	}
+			return std::vector<char>();
+		})
+	{}
 
 	bool tripped() const { return m_tripped; }
 
@@ -329,15 +365,8 @@ struct fake_node
 
 private:
 
-	std::array<char, 300> m_in_buffer;
-
 	std::vector<std::vector<char>> m_incoming_packets;
-
-	asio::io_service m_ios;
-	asio::ip::udp::socket m_socket;
-	bool m_tripped;
-
-	std::vector<char> m_send_buffer;
+	bool m_tripped = false;
 };
 
 template<unsigned long N>

simulation/libsimulator

@@ -1 +1 @@
-Subproject commit e450864958668f8c2ecf8b9839fa278c9c797571
+Subproject commit 36b46fc2c316d34714315c04c87cf74de6efae90

simulation/test_socks5.cpp

@@ -296,3 +296,90 @@ TORRENT_TEST(socks5_tcp_announce)
 	TEST_CHECK(tracker_port != -1);
 }
 
+TORRENT_TEST(udp_tracker)
+{
+	using namespace libtorrent;
+	bool tracker_alert = false;
+	bool connected = false;
+	bool announced = false;
+	run_test(
+		[](lt::session& ses)
+		{
+			set_proxy(ses, settings_pack::socks5);
+
+			// The socks server in libsimulator does not support forwarding UDP
+			// packets to hostnames (just IPv4 destinations)
+			settings_pack p;
+			p.set_bool(settings_pack::proxy_hostnames, false);
+			ses.apply_settings(p);
+
+			lt::add_torrent_params params;
+			params.info_hash = sha1_hash("abababababababababab");
+			params.trackers.push_back("udp://2.2.2.2:8080/announce");
+			params.save_path = ".";
+			ses.async_add_torrent(params);
+		},
+		[&tracker_alert](lt::session& ses, lt::alert const* alert) {
+			if (lt::alert_cast<lt::tracker_announce_alert>(alert))
+				tracker_alert = true;
+		},
+		[&](sim::simulation& sim, lt::session& ses
+			, boost::shared_ptr<lt::torrent_info> ti)
+		{
+			// listen on port 8080
+			udp_server tracker(sim, "2.2.2.2", 8080,
+			[&](char const* msg, int size)
+			{
+				using namespace libtorrent::detail;
+				std::vector<char> ret;
+				TEST_CHECK(size >= 16);
+
+				if (size < 16) return ret;
+
+				std::uint64_t connection_id = read_uint64(msg);
+				std::uint32_t action = read_uint32(msg);
+				std::uint32_t transaction_id = read_uint32(msg);
+
+				std::uint64_t const conn_id = 0xfeedface1337ull;
+
+				if (action == 0)
+				{
+					std::printf("udp connect\n");
+					// udp tracker connect
+					TEST_CHECK(connection_id == 0x41727101980ull);
+					auto inserter = std::back_inserter(ret);
+					write_uint32(0, inserter); // connect
+					write_uint32(transaction_id, inserter);
+					write_uint64(conn_id, inserter);
+					connected = true;
+				}
+				else if (action == 1)
+				{
+					std::printf("udp announce\n");
+					// udp tracker announce
+					TEST_EQUAL(connection_id, conn_id);
+
+					auto inserter = std::back_inserter(ret);
+					write_uint32(1, inserter); // announce
+					write_uint32(transaction_id, inserter);
+					write_uint32(1800, inserter);
+					write_uint32(0, inserter); // leechers
+					write_uint32(0, inserter); // seeders
+					announced = true;
+				}
+				else
+				{
+					std::printf("unsupported udp tracker action: %d\n", action);
+				}
+				return ret;
+			});
+
+			sim.run();
+		}
+	);
+
+	TEST_CHECK(tracker_alert);
+	TEST_CHECK(connected);
+	TEST_CHECK(announced);
+}
+

simulation/utils.cpp

@@ -114,6 +114,7 @@ void set_proxy(lt::session& ses, int proxy_type, int flags, bool proxy_peer_conn
 	p.set_bool(settings_pack::proxy_hostnames, true);
 	p.set_bool(settings_pack::proxy_peer_connections, proxy_peer_connections);
 	p.set_bool(settings_pack::proxy_tracker_connections, true);
+	p.set_bool(settings_pack::force_proxy, true);
 
 	ses.apply_settings(p);
 }

src/fingerprint.cpp

@@ -91,7 +91,7 @@ namespace libtorrent {
 #ifndef TORRENT_NO_DEPRECATE
 	std::string fingerprint::to_string() const
 	{
-		return generate_fingerprint(name, major_version, minor_version
+		return generate_fingerprint(std::string(name, 2), major_version, minor_version
 			, revision_version, tag_version);
 	}
 #endif // TORRENT_NO_DEPRECATE

src/peer_connection.cpp

@@ -1897,7 +1897,7 @@ namespace libtorrent
 		// calling disconnect_if_redundant, otherwise we may disconnect even if
 		// we are interested
 		if (!t->has_piece_passed(index)
-			&& !t->is_seed()
+			&& !t->is_upload_only()
 			&& !is_interesting()
 			&& (!t->has_picker() || t->picker().piece_priority(index) != 0))
 			t->peer_is_interesting(*this);

src/settings_pack.cpp

@@ -206,6 +206,7 @@ namespace libtorrent
 		SET(peer_timeout, 120, nullptr),
 		SET(urlseed_timeout, 20, nullptr),
 		SET(urlseed_pipeline_size, 5, nullptr),
+		SET(urlseed_max_request_bytes, 16 * 1024 * 1024, 0),
 		SET(urlseed_wait_retry, 30, nullptr),
 		SET(file_pool_size, 40, nullptr),
 		SET(max_failcount, 3, &session_impl::update_max_failcount),

src/torrent.cpp

@@ -4863,7 +4863,7 @@ namespace libtorrent
 		if (!has_picker())
 		{
 			pieces->clear();
-			pieces->resize(m_torrent_file->num_pieces(), 1);
+			pieces->resize(m_torrent_file->num_pieces(), 4);
 			return;
 		}
 
@@ -4936,8 +4936,8 @@ namespace libtorrent
 		else if (prio > 7) prio = 7;
 		if (int(m_file_priority.size()) <= index)
 		{
-			// any unallocated slot is assumed to be 1
-			if (prio == 1) return;
+			// any unallocated slot is assumed to be 4
+			if (prio == 4) return;
 			m_file_priority.resize(index + 1, 4);
 		}
 
@@ -5026,6 +5026,8 @@ namespace libtorrent
 
 			if (file_prio == 0)
 			{
+				// the pieces already start out as priority 0, no need to update
+				// the pieces vector in this case
 				need_update = true;
 				continue;
 			}
@@ -5042,8 +5044,7 @@ namespace libtorrent
 				, pieces.begin() + last_piece + 1
 				, std::bind(&set_if_greater, _1, file_prio));
 
-			if (has_picker() || file_prio != 1)
-				need_update = true;
+			need_update = true;
 		}
 		if (need_update) prioritize_pieces(pieces);
 	}

src/udp_socket.cpp

@@ -446,7 +446,9 @@ void udp_socket::bind(udp::endpoint const& ep, error_code& ec)
 	m_socket.io_control(ioc, ec);
 	if (ec) return;
 
-	m_bind_port = ep.port();
+	error_code err;
+	m_bind_port = m_socket.local_endpoint(err).port();
+	if (err) m_bind_port = ep.port();
 }
 
 void udp_socket::set_proxy_settings(aux::proxy_settings const& ps)

src/web_peer_connection.cpp

@@ -75,16 +75,16 @@ web_peer_connection::web_peer_connection(peer_connection_args const& pack
 	std::shared_ptr<torrent> tor = pack.tor.lock();
 	TORRENT_ASSERT(tor);
 
-	// we always prefer downloading 1 MiB chunks
-	// from web seeds, or whole pieces if pieces
-	// are larger than a MiB
-	int preferred_size = 1024 * 1024;
+	// if the web server is known not to support keep-alive. request 4MiB
+	// but we want to have at least piece size to prevent block based requests
+	int const min_size = std::max((web.supports_keepalive ? 1 : 4) * 1024 * 1024,
+		tor->torrent_file().piece_length());
 
-	// if the web server is known not to support keep-alive.
-	// request even larger blocks at a time
-	if (!web.supports_keepalive) preferred_size *= 4;
+	// we prefer downloading large chunks from web seeds,
+	// but still want to be able to split requests
+	int const preferred_size = std::max(min_size, m_settings.get_int(settings_pack::urlseed_max_request_bytes));
 
-	prefer_contiguous_blocks((std::max)(preferred_size / tor->block_size(), 1));
+	prefer_contiguous_blocks(preferred_size / tor->block_size());
 
 	std::shared_ptr<torrent> t = associated_torrent().lock();
 	bool const single_file_request = t->torrent_file().num_files() == 1;