diff --git a/ChangeLog b/ChangeLog index 746aef87b..cd39431d6 100644 --- a/ChangeLog +++ b/ChangeLog @@ -46,6 +46,9 @@ * resume data no longer has timestamps of files * require C++11 to build libtorrent + * fix socks5 support for UDP + * add setting urlseed_max_request_bytes to handle large web seed requests + * fix python build with CC/CXX environment * add trackers from add_torrent_params/magnet links to separate tiers * fix resumedata check issue with files with priority 0 * deprecated mmap_cache feature diff --git a/bindings/python/setup.py b/bindings/python/setup.py index f42ee59f5..ee00478dd 100644 --- a/bindings/python/setup.py +++ b/bindings/python/setup.py @@ -114,7 +114,10 @@ else: # for some reason distutils uses the CC environment variable to determine # the compiler to use for C++ - os.environ["CC"] = os.environ['CXX'] + if 'CXX' in os.environ: + os.environ['CC'] = os.environ['CXX'] + if 'CXXFLAGS' in os.environ: + os.environ['CFLAGS'] = os.environ['CXXFLAGS'] ext = [Extension('libtorrent', sources = source_list, diff --git a/bindings/python/src/session.cpp b/bindings/python/src/session.cpp index 231c61185..80b009b91 100644 --- a/bindings/python/src/session.cpp +++ b/bindings/python/src/session.cpp @@ -357,7 +357,7 @@ namespace d["piece"] = i->piece; d["last_use"] = total_milliseconds(now - i->last_use) / 1000.f; d["next_to_hash"] = i->next_to_hash; - d["kind"] = i->kind; + d["kind"] = static_cast(i->kind); pieces.append(d); } return pieces; diff --git a/bindings/python/test.py b/bindings/python/test.py index 886d4b80b..5231b6e10 100644 --- a/bindings/python/test.py +++ b/bindings/python/test.py @@ -29,6 +29,9 @@ class test_torrent_handle(unittest.TestCase): ti = lt.torrent_info('url_seed_multi.torrent'); h = ses.add_torrent({'ti': ti, 'save_path': os.getcwd()}) + self.assertEqual(h.file_priorities(), [4,4]) + self.assertEqual(h.piece_priorities(), [4]) + h.prioritize_files([0,1]) self.assertEqual(h.file_priorities(), [0,1]) diff --git a/include/libtorrent/peer_connection.hpp b/include/libtorrent/peer_connection.hpp index ec7d08a0c..8dde4efa9 100644 --- a/include/libtorrent/peer_connection.hpp +++ b/include/libtorrent/peer_connection.hpp @@ -334,7 +334,7 @@ namespace libtorrent int picker_options() const; void prefer_contiguous_blocks(int num) - { m_prefer_contiguous_blocks = std::uint8_t((std::min)(num, 255)); } + { m_prefer_contiguous_blocks = num; } bool request_large_blocks() const { return m_request_large_blocks; } @@ -1066,7 +1066,7 @@ namespace libtorrent // if it is 0, the download rate limit setting // will be used to determine if whole pieces // are preferred. - std::uint8_t m_prefer_contiguous_blocks = 0; + int m_prefer_contiguous_blocks = 0; // this is the number of times this peer has had // a request rejected because of a disk I/O failure. diff --git a/include/libtorrent/settings_pack.hpp b/include/libtorrent/settings_pack.hpp index 53ce9832c..05c80d326 100644 --- a/include/libtorrent/settings_pack.hpp +++ b/include/libtorrent/settings_pack.hpp @@ -780,6 +780,21 @@ namespace libtorrent // low number, like 5 urlseed_pipeline_size, + // The maximum request range of an url seed in bytes. This value + // defines the largest possible sequential web seed request. Default + // is 16 * 1024 * 1024. Lower values are possible but will be ignored + // if they are lower then piece size. + // This value should be related to your download speed to prevent + // libtorrent from creating too many expensive http requests per + // second. You can select a value as high as you want but keep in mind + // that libtorrent can't create parallel requests if the first request + // did already select the whole file. + // If you combine bittorrent seeds with web seeds and pick strategies + // like rarest first you may find your web seed requests split into + // smaller parts because we don't download already picked pieces + // twice. + urlseed_max_request_bytes, + // time to wait until a new retry of a web seed takes place urlseed_wait_retry, diff --git a/include/libtorrent/torrent_handle.hpp b/include/libtorrent/torrent_handle.hpp index 5f2565675..ac22206a5 100644 --- a/include/libtorrent/torrent_handle.hpp +++ b/include/libtorrent/torrent_handle.hpp @@ -961,8 +961,6 @@ namespace libtorrent // will return the resume data in an alert TORRENT_DEPRECATED entry write_resume_data() const; - // ================ end deprecation ============ -#endif // ``use_interface()`` sets the network interface this torrent will use // when it opens outgoing connections. By default, it uses the same @@ -971,7 +969,10 @@ namespace libtorrent // IPv4 or IPv6 address). When specifying multiple interfaces, the // torrent will round-robin which interface to use for each outgoing // connection. This is useful for clients that are multi-homed. + TORRENT_DEPRECATED void use_interface(const char* net_interface) const; + // ================ end deprecation ============ +#endif // Fills the specified ``std::vector`` with the availability for // each piece in this torrent. libtorrent does not keep track of diff --git a/m4/ax_check_openssl.m4 b/m4/ax_check_openssl.m4 index aecf20849..973f7e880 100644 --- a/m4/ax_check_openssl.m4 +++ b/m4/ax_check_openssl.m4 @@ -106,7 +106,7 @@ AC_DEFUN([AX_CHECK_OPENSSL], [ LIBS="$OPENSSL_LIBS $LIBS" CPPFLAGS="$OPENSSL_INCLUDES $CPPFLAGS" AC_LINK_IFELSE( - AC_LANG_PROGRAM([#include ], [SSL_new(NULL)]), + [AC_LANG_PROGRAM([#include ], [SSL_new(NULL)])], [ AC_MSG_RESULT([yes]) $1 diff --git a/simulation/fake_peer.hpp b/simulation/fake_peer.hpp index dd3ad784c..9e4d875f8 100644 --- a/simulation/fake_peer.hpp +++ b/simulation/fake_peer.hpp @@ -55,12 +55,6 @@ struct fake_peer { fake_peer(simulation& sim, char const* ip) : m_ios(sim, asio::ip::address::from_string(ip)) - , m_acceptor(m_ios) - , m_socket(m_ios) - , m_info_hash(0) - , m_accepted(false) - , m_connected(false) - , m_disconnected(false) { boost::system::error_code ec; m_acceptor.open(asio::ip::tcp::v4(), ec); @@ -249,19 +243,19 @@ private: char m_out_buffer[300]; asio::io_service m_ios; - asio::ip::tcp::acceptor m_acceptor; - asio::ip::tcp::socket m_socket; + asio::ip::tcp::acceptor m_acceptor{m_ios}; + asio::ip::tcp::socket m_socket{m_ios}; lt::sha1_hash m_info_hash; // set to true if this peer received an incoming connection // if this is an outgoing connection, this will always be false - bool m_accepted; + bool m_accepted = false; // set to true if this peer completed a bittorrent handshake - bool m_connected; + bool m_connected = false; // set to true if this peer has been disconnected by the other end - bool m_disconnected; + bool m_disconnected = false; std::vector m_send_buffer; }; @@ -283,12 +277,12 @@ inline void add_fake_peers(lt::torrent_handle& h, int const n = 5) } } -struct fake_node +struct udp_server { - fake_node(simulation& sim, char const* ip, int port = 6881) + udp_server(simulation& sim, char const* ip, int port + , std::function(char const*, int)> handler) : m_ios(sim, asio::ip::address::from_string(ip)) - , m_socket(m_ios) - , m_tripped(false) + , m_handler(handler) { boost::system::error_code ec; m_socket.open(asio::ip::udp::v4(), ec); @@ -296,31 +290,73 @@ struct fake_node m_socket.bind(asio::ip::udp::endpoint(asio::ip::address_v4::any(), port), ec); TEST_CHECK(!ec); - std::printf("fake_node::async_read_some\n"); - m_socket.async_receive(boost::asio::buffer(m_in_buffer) - , [&] (boost::system::error_code const& ec, size_t bytes_transferred) - { - std::printf("fake_node::async_read_some callback. ec: %s transferred: %d\n" - , ec.message().c_str(), int(bytes_transferred)); - if (ec) return; + m_socket.io_control(lt::udp::socket::non_blocking_io(true)); + std::printf("udp_server::async_read_some\n"); + using namespace std::placeholders; + m_socket.async_receive_from(boost::asio::buffer(m_in_buffer) + , m_from, 0, std::bind(&udp_server::on_read, this, _1, _2)); + } + + void close() { m_socket.close(); } + +private: + + void on_read(boost::system::error_code const& ec, size_t bytes_transferred) + { + std::printf("udp_server::async_read_some callback. ec: %s transferred: %d\n" + , ec.message().c_str(), int(bytes_transferred)); + if (ec) return; + + std::vector send_buffer = m_handler(m_in_buffer.data(), int(bytes_transferred)); + + if (!send_buffer.empty()) + { + lt::error_code err; + m_socket.send_to(boost::asio::buffer(send_buffer), m_from, 0, err); + if (err) + { + std::printf("send_to FAILED: %s\n", err.message().c_str()); + } + else + { + std::printf("udp_server responding with %d bytes\n" + , int(send_buffer.size())); + } + } + + std::printf("udp_server::async_read_some\n"); + using namespace std::placeholders; + m_socket.async_receive_from(boost::asio::buffer(m_in_buffer) + , m_from, 0, std::bind(&udp_server::on_read, this, _1, _2)); + } + + std::array m_in_buffer; + + asio::io_service m_ios; + asio::ip::udp::socket m_socket{m_ios}; + asio::ip::udp::endpoint m_from; + + std::function(char const*, int)> m_handler; +}; + +struct fake_node : udp_server +{ + fake_node(simulation& sim, char const* ip, int port = 6881) + : udp_server(sim, ip, port, [&](char const* incoming, int size) + { lt::bdecode_node n; boost::system::error_code err; - int const ret = bdecode(m_in_buffer.data(), m_in_buffer.data() + bytes_transferred - , n, err, nullptr, 10, 200); + int const ret = bdecode(incoming, incoming + size, n, err, nullptr, 10, 200); TEST_EQUAL(ret, 0); - m_incoming_packets.emplace_back(m_in_buffer.data(), m_in_buffer.data() + bytes_transferred); + m_incoming_packets.emplace_back(incoming, incoming + size); // TODO: ideally we would validate the DHT message m_tripped = true; - }); - } - - void close() - { - m_socket.close(); - } + return std::vector(); + }) + {} bool tripped() const { return m_tripped; } @@ -329,15 +365,8 @@ struct fake_node private: - std::array m_in_buffer; - std::vector> m_incoming_packets; - - asio::io_service m_ios; - asio::ip::udp::socket m_socket; - bool m_tripped; - - std::vector m_send_buffer; + bool m_tripped = false; }; template diff --git a/simulation/libsimulator b/simulation/libsimulator index e45086495..36b46fc2c 160000 --- a/simulation/libsimulator +++ b/simulation/libsimulator @@ -1 +1 @@ -Subproject commit e450864958668f8c2ecf8b9839fa278c9c797571 +Subproject commit 36b46fc2c316d34714315c04c87cf74de6efae90 diff --git a/simulation/test_socks5.cpp b/simulation/test_socks5.cpp index 6b37d5802..439308b54 100644 --- a/simulation/test_socks5.cpp +++ b/simulation/test_socks5.cpp @@ -296,3 +296,90 @@ TORRENT_TEST(socks5_tcp_announce) TEST_CHECK(tracker_port != -1); } +TORRENT_TEST(udp_tracker) +{ + using namespace libtorrent; + bool tracker_alert = false; + bool connected = false; + bool announced = false; + run_test( + [](lt::session& ses) + { + set_proxy(ses, settings_pack::socks5); + + // The socks server in libsimulator does not support forwarding UDP + // packets to hostnames (just IPv4 destinations) + settings_pack p; + p.set_bool(settings_pack::proxy_hostnames, false); + ses.apply_settings(p); + + lt::add_torrent_params params; + params.info_hash = sha1_hash("abababababababababab"); + params.trackers.push_back("udp://2.2.2.2:8080/announce"); + params.save_path = "."; + ses.async_add_torrent(params); + }, + [&tracker_alert](lt::session& ses, lt::alert const* alert) { + if (lt::alert_cast(alert)) + tracker_alert = true; + }, + [&](sim::simulation& sim, lt::session& ses + , boost::shared_ptr ti) + { + // listen on port 8080 + udp_server tracker(sim, "2.2.2.2", 8080, + [&](char const* msg, int size) + { + using namespace libtorrent::detail; + std::vector ret; + TEST_CHECK(size >= 16); + + if (size < 16) return ret; + + std::uint64_t connection_id = read_uint64(msg); + std::uint32_t action = read_uint32(msg); + std::uint32_t transaction_id = read_uint32(msg); + + std::uint64_t const conn_id = 0xfeedface1337ull; + + if (action == 0) + { + std::printf("udp connect\n"); + // udp tracker connect + TEST_CHECK(connection_id == 0x41727101980ull); + auto inserter = std::back_inserter(ret); + write_uint32(0, inserter); // connect + write_uint32(transaction_id, inserter); + write_uint64(conn_id, inserter); + connected = true; + } + else if (action == 1) + { + std::printf("udp announce\n"); + // udp tracker announce + TEST_EQUAL(connection_id, conn_id); + + auto inserter = std::back_inserter(ret); + write_uint32(1, inserter); // announce + write_uint32(transaction_id, inserter); + write_uint32(1800, inserter); + write_uint32(0, inserter); // leechers + write_uint32(0, inserter); // seeders + announced = true; + } + else + { + std::printf("unsupported udp tracker action: %d\n", action); + } + return ret; + }); + + sim.run(); + } + ); + + TEST_CHECK(tracker_alert); + TEST_CHECK(connected); + TEST_CHECK(announced); +} + diff --git a/simulation/utils.cpp b/simulation/utils.cpp index 0905c9988..d838c8814 100644 --- a/simulation/utils.cpp +++ b/simulation/utils.cpp @@ -114,6 +114,7 @@ void set_proxy(lt::session& ses, int proxy_type, int flags, bool proxy_peer_conn p.set_bool(settings_pack::proxy_hostnames, true); p.set_bool(settings_pack::proxy_peer_connections, proxy_peer_connections); p.set_bool(settings_pack::proxy_tracker_connections, true); + p.set_bool(settings_pack::force_proxy, true); ses.apply_settings(p); } diff --git a/src/fingerprint.cpp b/src/fingerprint.cpp index d953dfb86..5c1af5f36 100644 --- a/src/fingerprint.cpp +++ b/src/fingerprint.cpp @@ -91,7 +91,7 @@ namespace libtorrent { #ifndef TORRENT_NO_DEPRECATE std::string fingerprint::to_string() const { - return generate_fingerprint(name, major_version, minor_version + return generate_fingerprint(std::string(name, 2), major_version, minor_version , revision_version, tag_version); } #endif // TORRENT_NO_DEPRECATE diff --git a/src/peer_connection.cpp b/src/peer_connection.cpp index b54d893a9..2ec8c4890 100644 --- a/src/peer_connection.cpp +++ b/src/peer_connection.cpp @@ -1897,7 +1897,7 @@ namespace libtorrent // calling disconnect_if_redundant, otherwise we may disconnect even if // we are interested if (!t->has_piece_passed(index) - && !t->is_seed() + && !t->is_upload_only() && !is_interesting() && (!t->has_picker() || t->picker().piece_priority(index) != 0)) t->peer_is_interesting(*this); diff --git a/src/settings_pack.cpp b/src/settings_pack.cpp index 20fcee691..a63948646 100644 --- a/src/settings_pack.cpp +++ b/src/settings_pack.cpp @@ -206,6 +206,7 @@ namespace libtorrent SET(peer_timeout, 120, nullptr), SET(urlseed_timeout, 20, nullptr), SET(urlseed_pipeline_size, 5, nullptr), + SET(urlseed_max_request_bytes, 16 * 1024 * 1024, 0), SET(urlseed_wait_retry, 30, nullptr), SET(file_pool_size, 40, nullptr), SET(max_failcount, 3, &session_impl::update_max_failcount), diff --git a/src/torrent.cpp b/src/torrent.cpp index ae4c4f7f3..16d29e7d3 100644 --- a/src/torrent.cpp +++ b/src/torrent.cpp @@ -4863,7 +4863,7 @@ namespace libtorrent if (!has_picker()) { pieces->clear(); - pieces->resize(m_torrent_file->num_pieces(), 1); + pieces->resize(m_torrent_file->num_pieces(), 4); return; } @@ -4936,8 +4936,8 @@ namespace libtorrent else if (prio > 7) prio = 7; if (int(m_file_priority.size()) <= index) { - // any unallocated slot is assumed to be 1 - if (prio == 1) return; + // any unallocated slot is assumed to be 4 + if (prio == 4) return; m_file_priority.resize(index + 1, 4); } @@ -5026,6 +5026,8 @@ namespace libtorrent if (file_prio == 0) { + // the pieces already start out as priority 0, no need to update + // the pieces vector in this case need_update = true; continue; } @@ -5042,8 +5044,7 @@ namespace libtorrent , pieces.begin() + last_piece + 1 , std::bind(&set_if_greater, _1, file_prio)); - if (has_picker() || file_prio != 1) - need_update = true; + need_update = true; } if (need_update) prioritize_pieces(pieces); } diff --git a/src/udp_socket.cpp b/src/udp_socket.cpp index d453b21eb..4afc33fdc 100644 --- a/src/udp_socket.cpp +++ b/src/udp_socket.cpp @@ -446,7 +446,9 @@ void udp_socket::bind(udp::endpoint const& ep, error_code& ec) m_socket.io_control(ioc, ec); if (ec) return; - m_bind_port = ep.port(); + error_code err; + m_bind_port = m_socket.local_endpoint(err).port(); + if (err) m_bind_port = ep.port(); } void udp_socket::set_proxy_settings(aux::proxy_settings const& ps) diff --git a/src/web_peer_connection.cpp b/src/web_peer_connection.cpp index 62e118ad2..23bbf274b 100644 --- a/src/web_peer_connection.cpp +++ b/src/web_peer_connection.cpp @@ -75,16 +75,16 @@ web_peer_connection::web_peer_connection(peer_connection_args const& pack std::shared_ptr tor = pack.tor.lock(); TORRENT_ASSERT(tor); - // we always prefer downloading 1 MiB chunks - // from web seeds, or whole pieces if pieces - // are larger than a MiB - int preferred_size = 1024 * 1024; + // if the web server is known not to support keep-alive. request 4MiB + // but we want to have at least piece size to prevent block based requests + int const min_size = std::max((web.supports_keepalive ? 1 : 4) * 1024 * 1024, + tor->torrent_file().piece_length()); - // if the web server is known not to support keep-alive. - // request even larger blocks at a time - if (!web.supports_keepalive) preferred_size *= 4; + // we prefer downloading large chunks from web seeds, + // but still want to be able to split requests + int const preferred_size = std::max(min_size, m_settings.get_int(settings_pack::urlseed_max_request_bytes)); - prefer_contiguous_blocks((std::max)(preferred_size / tor->block_size(), 1)); + prefer_contiguous_blocks(preferred_size / tor->block_size()); std::shared_ptr t = associated_torrent().lock(); bool const single_file_request = t->torrent_file().num_files() == 1;