merged RC_1_1 into master

This commit is contained in:
arvidn 2016-12-20 01:16:13 -05:00
commit a2c4e7490e
18 changed files with 210 additions and 64 deletions

View File

@ -46,6 +46,9 @@
* resume data no longer has timestamps of files * resume data no longer has timestamps of files
* require C++11 to build libtorrent * require C++11 to build libtorrent
* fix socks5 support for UDP
* add setting urlseed_max_request_bytes to handle large web seed requests
* fix python build with CC/CXX environment
* add trackers from add_torrent_params/magnet links to separate tiers * add trackers from add_torrent_params/magnet links to separate tiers
* fix resumedata check issue with files with priority 0 * fix resumedata check issue with files with priority 0
* deprecated mmap_cache feature * deprecated mmap_cache feature

View File

@ -114,7 +114,10 @@ else:
# for some reason distutils uses the CC environment variable to determine # for some reason distutils uses the CC environment variable to determine
# the compiler to use for C++ # the compiler to use for C++
os.environ["CC"] = os.environ['CXX'] if 'CXX' in os.environ:
os.environ['CC'] = os.environ['CXX']
if 'CXXFLAGS' in os.environ:
os.environ['CFLAGS'] = os.environ['CXXFLAGS']
ext = [Extension('libtorrent', ext = [Extension('libtorrent',
sources = source_list, sources = source_list,

View File

@ -357,7 +357,7 @@ namespace
d["piece"] = i->piece; d["piece"] = i->piece;
d["last_use"] = total_milliseconds(now - i->last_use) / 1000.f; d["last_use"] = total_milliseconds(now - i->last_use) / 1000.f;
d["next_to_hash"] = i->next_to_hash; d["next_to_hash"] = i->next_to_hash;
d["kind"] = i->kind; d["kind"] = static_cast<int>(i->kind);
pieces.append(d); pieces.append(d);
} }
return pieces; return pieces;

View File

@ -29,6 +29,9 @@ class test_torrent_handle(unittest.TestCase):
ti = lt.torrent_info('url_seed_multi.torrent'); ti = lt.torrent_info('url_seed_multi.torrent');
h = ses.add_torrent({'ti': ti, 'save_path': os.getcwd()}) h = ses.add_torrent({'ti': ti, 'save_path': os.getcwd()})
self.assertEqual(h.file_priorities(), [4,4])
self.assertEqual(h.piece_priorities(), [4])
h.prioritize_files([0,1]) h.prioritize_files([0,1])
self.assertEqual(h.file_priorities(), [0,1]) self.assertEqual(h.file_priorities(), [0,1])

View File

@ -334,7 +334,7 @@ namespace libtorrent
int picker_options() const; int picker_options() const;
void prefer_contiguous_blocks(int num) void prefer_contiguous_blocks(int num)
{ m_prefer_contiguous_blocks = std::uint8_t((std::min)(num, 255)); } { m_prefer_contiguous_blocks = num; }
bool request_large_blocks() const bool request_large_blocks() const
{ return m_request_large_blocks; } { return m_request_large_blocks; }
@ -1066,7 +1066,7 @@ namespace libtorrent
// if it is 0, the download rate limit setting // if it is 0, the download rate limit setting
// will be used to determine if whole pieces // will be used to determine if whole pieces
// are preferred. // are preferred.
std::uint8_t m_prefer_contiguous_blocks = 0; int m_prefer_contiguous_blocks = 0;
// this is the number of times this peer has had // this is the number of times this peer has had
// a request rejected because of a disk I/O failure. // a request rejected because of a disk I/O failure.

View File

@ -780,6 +780,21 @@ namespace libtorrent
// low number, like 5 // low number, like 5
urlseed_pipeline_size, urlseed_pipeline_size,
// The maximum request range of an url seed in bytes. This value
// defines the largest possible sequential web seed request. Default
// is 16 * 1024 * 1024. Lower values are possible but will be ignored
// if they are lower then piece size.
// This value should be related to your download speed to prevent
// libtorrent from creating too many expensive http requests per
// second. You can select a value as high as you want but keep in mind
// that libtorrent can't create parallel requests if the first request
// did already select the whole file.
// If you combine bittorrent seeds with web seeds and pick strategies
// like rarest first you may find your web seed requests split into
// smaller parts because we don't download already picked pieces
// twice.
urlseed_max_request_bytes,
// time to wait until a new retry of a web seed takes place // time to wait until a new retry of a web seed takes place
urlseed_wait_retry, urlseed_wait_retry,

View File

@ -961,8 +961,6 @@ namespace libtorrent
// will return the resume data in an alert // will return the resume data in an alert
TORRENT_DEPRECATED TORRENT_DEPRECATED
entry write_resume_data() const; entry write_resume_data() const;
// ================ end deprecation ============
#endif
// ``use_interface()`` sets the network interface this torrent will use // ``use_interface()`` sets the network interface this torrent will use
// when it opens outgoing connections. By default, it uses the same // when it opens outgoing connections. By default, it uses the same
@ -971,7 +969,10 @@ namespace libtorrent
// IPv4 or IPv6 address). When specifying multiple interfaces, the // IPv4 or IPv6 address). When specifying multiple interfaces, the
// torrent will round-robin which interface to use for each outgoing // torrent will round-robin which interface to use for each outgoing
// connection. This is useful for clients that are multi-homed. // connection. This is useful for clients that are multi-homed.
TORRENT_DEPRECATED
void use_interface(const char* net_interface) const; void use_interface(const char* net_interface) const;
// ================ end deprecation ============
#endif
// Fills the specified ``std::vector<int>`` with the availability for // Fills the specified ``std::vector<int>`` with the availability for
// each piece in this torrent. libtorrent does not keep track of // each piece in this torrent. libtorrent does not keep track of

View File

@ -106,7 +106,7 @@ AC_DEFUN([AX_CHECK_OPENSSL], [
LIBS="$OPENSSL_LIBS $LIBS" LIBS="$OPENSSL_LIBS $LIBS"
CPPFLAGS="$OPENSSL_INCLUDES $CPPFLAGS" CPPFLAGS="$OPENSSL_INCLUDES $CPPFLAGS"
AC_LINK_IFELSE( AC_LINK_IFELSE(
AC_LANG_PROGRAM([#include <openssl/ssl.h>], [SSL_new(NULL)]), [AC_LANG_PROGRAM([#include <openssl/ssl.h>], [SSL_new(NULL)])],
[ [
AC_MSG_RESULT([yes]) AC_MSG_RESULT([yes])
$1 $1

View File

@ -55,12 +55,6 @@ struct fake_peer
{ {
fake_peer(simulation& sim, char const* ip) fake_peer(simulation& sim, char const* ip)
: m_ios(sim, asio::ip::address::from_string(ip)) : m_ios(sim, asio::ip::address::from_string(ip))
, m_acceptor(m_ios)
, m_socket(m_ios)
, m_info_hash(0)
, m_accepted(false)
, m_connected(false)
, m_disconnected(false)
{ {
boost::system::error_code ec; boost::system::error_code ec;
m_acceptor.open(asio::ip::tcp::v4(), ec); m_acceptor.open(asio::ip::tcp::v4(), ec);
@ -249,19 +243,19 @@ private:
char m_out_buffer[300]; char m_out_buffer[300];
asio::io_service m_ios; asio::io_service m_ios;
asio::ip::tcp::acceptor m_acceptor; asio::ip::tcp::acceptor m_acceptor{m_ios};
asio::ip::tcp::socket m_socket; asio::ip::tcp::socket m_socket{m_ios};
lt::sha1_hash m_info_hash; lt::sha1_hash m_info_hash;
// set to true if this peer received an incoming connection // set to true if this peer received an incoming connection
// if this is an outgoing connection, this will always be false // if this is an outgoing connection, this will always be false
bool m_accepted; bool m_accepted = false;
// set to true if this peer completed a bittorrent handshake // set to true if this peer completed a bittorrent handshake
bool m_connected; bool m_connected = false;
// set to true if this peer has been disconnected by the other end // set to true if this peer has been disconnected by the other end
bool m_disconnected; bool m_disconnected = false;
std::vector<char> m_send_buffer; std::vector<char> m_send_buffer;
}; };
@ -283,12 +277,12 @@ inline void add_fake_peers(lt::torrent_handle& h, int const n = 5)
} }
} }
struct fake_node struct udp_server
{ {
fake_node(simulation& sim, char const* ip, int port = 6881) udp_server(simulation& sim, char const* ip, int port
, std::function<std::vector<char>(char const*, int)> handler)
: m_ios(sim, asio::ip::address::from_string(ip)) : m_ios(sim, asio::ip::address::from_string(ip))
, m_socket(m_ios) , m_handler(handler)
, m_tripped(false)
{ {
boost::system::error_code ec; boost::system::error_code ec;
m_socket.open(asio::ip::udp::v4(), ec); m_socket.open(asio::ip::udp::v4(), ec);
@ -296,31 +290,73 @@ struct fake_node
m_socket.bind(asio::ip::udp::endpoint(asio::ip::address_v4::any(), port), ec); m_socket.bind(asio::ip::udp::endpoint(asio::ip::address_v4::any(), port), ec);
TEST_CHECK(!ec); TEST_CHECK(!ec);
std::printf("fake_node::async_read_some\n"); m_socket.io_control(lt::udp::socket::non_blocking_io(true));
m_socket.async_receive(boost::asio::buffer(m_in_buffer)
, [&] (boost::system::error_code const& ec, size_t bytes_transferred) std::printf("udp_server::async_read_some\n");
using namespace std::placeholders;
m_socket.async_receive_from(boost::asio::buffer(m_in_buffer)
, m_from, 0, std::bind(&udp_server::on_read, this, _1, _2));
}
void close() { m_socket.close(); }
private:
void on_read(boost::system::error_code const& ec, size_t bytes_transferred)
{ {
std::printf("fake_node::async_read_some callback. ec: %s transferred: %d\n" std::printf("udp_server::async_read_some callback. ec: %s transferred: %d\n"
, ec.message().c_str(), int(bytes_transferred)); , ec.message().c_str(), int(bytes_transferred));
if (ec) return; if (ec) return;
std::vector<char> send_buffer = m_handler(m_in_buffer.data(), int(bytes_transferred));
if (!send_buffer.empty())
{
lt::error_code err;
m_socket.send_to(boost::asio::buffer(send_buffer), m_from, 0, err);
if (err)
{
std::printf("send_to FAILED: %s\n", err.message().c_str());
}
else
{
std::printf("udp_server responding with %d bytes\n"
, int(send_buffer.size()));
}
}
std::printf("udp_server::async_read_some\n");
using namespace std::placeholders;
m_socket.async_receive_from(boost::asio::buffer(m_in_buffer)
, m_from, 0, std::bind(&udp_server::on_read, this, _1, _2));
}
std::array<char, 1500> m_in_buffer;
asio::io_service m_ios;
asio::ip::udp::socket m_socket{m_ios};
asio::ip::udp::endpoint m_from;
std::function<std::vector<char>(char const*, int)> m_handler;
};
struct fake_node : udp_server
{
fake_node(simulation& sim, char const* ip, int port = 6881)
: udp_server(sim, ip, port, [&](char const* incoming, int size)
{
lt::bdecode_node n; lt::bdecode_node n;
boost::system::error_code err; boost::system::error_code err;
int const ret = bdecode(m_in_buffer.data(), m_in_buffer.data() + bytes_transferred int const ret = bdecode(incoming, incoming + size, n, err, nullptr, 10, 200);
, n, err, nullptr, 10, 200);
TEST_EQUAL(ret, 0); TEST_EQUAL(ret, 0);
m_incoming_packets.emplace_back(m_in_buffer.data(), m_in_buffer.data() + bytes_transferred); m_incoming_packets.emplace_back(incoming, incoming + size);
// TODO: ideally we would validate the DHT message // TODO: ideally we would validate the DHT message
m_tripped = true; m_tripped = true;
}); return std::vector<char>();
} })
{}
void close()
{
m_socket.close();
}
bool tripped() const { return m_tripped; } bool tripped() const { return m_tripped; }
@ -329,15 +365,8 @@ struct fake_node
private: private:
std::array<char, 300> m_in_buffer;
std::vector<std::vector<char>> m_incoming_packets; std::vector<std::vector<char>> m_incoming_packets;
bool m_tripped = false;
asio::io_service m_ios;
asio::ip::udp::socket m_socket;
bool m_tripped;
std::vector<char> m_send_buffer;
}; };
template<unsigned long N> template<unsigned long N>

@ -1 +1 @@
Subproject commit e450864958668f8c2ecf8b9839fa278c9c797571 Subproject commit 36b46fc2c316d34714315c04c87cf74de6efae90

View File

@ -296,3 +296,90 @@ TORRENT_TEST(socks5_tcp_announce)
TEST_CHECK(tracker_port != -1); TEST_CHECK(tracker_port != -1);
} }
TORRENT_TEST(udp_tracker)
{
using namespace libtorrent;
bool tracker_alert = false;
bool connected = false;
bool announced = false;
run_test(
[](lt::session& ses)
{
set_proxy(ses, settings_pack::socks5);
// The socks server in libsimulator does not support forwarding UDP
// packets to hostnames (just IPv4 destinations)
settings_pack p;
p.set_bool(settings_pack::proxy_hostnames, false);
ses.apply_settings(p);
lt::add_torrent_params params;
params.info_hash = sha1_hash("abababababababababab");
params.trackers.push_back("udp://2.2.2.2:8080/announce");
params.save_path = ".";
ses.async_add_torrent(params);
},
[&tracker_alert](lt::session& ses, lt::alert const* alert) {
if (lt::alert_cast<lt::tracker_announce_alert>(alert))
tracker_alert = true;
},
[&](sim::simulation& sim, lt::session& ses
, boost::shared_ptr<lt::torrent_info> ti)
{
// listen on port 8080
udp_server tracker(sim, "2.2.2.2", 8080,
[&](char const* msg, int size)
{
using namespace libtorrent::detail;
std::vector<char> ret;
TEST_CHECK(size >= 16);
if (size < 16) return ret;
std::uint64_t connection_id = read_uint64(msg);
std::uint32_t action = read_uint32(msg);
std::uint32_t transaction_id = read_uint32(msg);
std::uint64_t const conn_id = 0xfeedface1337ull;
if (action == 0)
{
std::printf("udp connect\n");
// udp tracker connect
TEST_CHECK(connection_id == 0x41727101980ull);
auto inserter = std::back_inserter(ret);
write_uint32(0, inserter); // connect
write_uint32(transaction_id, inserter);
write_uint64(conn_id, inserter);
connected = true;
}
else if (action == 1)
{
std::printf("udp announce\n");
// udp tracker announce
TEST_EQUAL(connection_id, conn_id);
auto inserter = std::back_inserter(ret);
write_uint32(1, inserter); // announce
write_uint32(transaction_id, inserter);
write_uint32(1800, inserter);
write_uint32(0, inserter); // leechers
write_uint32(0, inserter); // seeders
announced = true;
}
else
{
std::printf("unsupported udp tracker action: %d\n", action);
}
return ret;
});
sim.run();
}
);
TEST_CHECK(tracker_alert);
TEST_CHECK(connected);
TEST_CHECK(announced);
}

View File

@ -114,6 +114,7 @@ void set_proxy(lt::session& ses, int proxy_type, int flags, bool proxy_peer_conn
p.set_bool(settings_pack::proxy_hostnames, true); p.set_bool(settings_pack::proxy_hostnames, true);
p.set_bool(settings_pack::proxy_peer_connections, proxy_peer_connections); p.set_bool(settings_pack::proxy_peer_connections, proxy_peer_connections);
p.set_bool(settings_pack::proxy_tracker_connections, true); p.set_bool(settings_pack::proxy_tracker_connections, true);
p.set_bool(settings_pack::force_proxy, true);
ses.apply_settings(p); ses.apply_settings(p);
} }

View File

@ -91,7 +91,7 @@ namespace libtorrent {
#ifndef TORRENT_NO_DEPRECATE #ifndef TORRENT_NO_DEPRECATE
std::string fingerprint::to_string() const std::string fingerprint::to_string() const
{ {
return generate_fingerprint(name, major_version, minor_version return generate_fingerprint(std::string(name, 2), major_version, minor_version
, revision_version, tag_version); , revision_version, tag_version);
} }
#endif // TORRENT_NO_DEPRECATE #endif // TORRENT_NO_DEPRECATE

View File

@ -1897,7 +1897,7 @@ namespace libtorrent
// calling disconnect_if_redundant, otherwise we may disconnect even if // calling disconnect_if_redundant, otherwise we may disconnect even if
// we are interested // we are interested
if (!t->has_piece_passed(index) if (!t->has_piece_passed(index)
&& !t->is_seed() && !t->is_upload_only()
&& !is_interesting() && !is_interesting()
&& (!t->has_picker() || t->picker().piece_priority(index) != 0)) && (!t->has_picker() || t->picker().piece_priority(index) != 0))
t->peer_is_interesting(*this); t->peer_is_interesting(*this);

View File

@ -206,6 +206,7 @@ namespace libtorrent
SET(peer_timeout, 120, nullptr), SET(peer_timeout, 120, nullptr),
SET(urlseed_timeout, 20, nullptr), SET(urlseed_timeout, 20, nullptr),
SET(urlseed_pipeline_size, 5, nullptr), SET(urlseed_pipeline_size, 5, nullptr),
SET(urlseed_max_request_bytes, 16 * 1024 * 1024, 0),
SET(urlseed_wait_retry, 30, nullptr), SET(urlseed_wait_retry, 30, nullptr),
SET(file_pool_size, 40, nullptr), SET(file_pool_size, 40, nullptr),
SET(max_failcount, 3, &session_impl::update_max_failcount), SET(max_failcount, 3, &session_impl::update_max_failcount),

View File

@ -4863,7 +4863,7 @@ namespace libtorrent
if (!has_picker()) if (!has_picker())
{ {
pieces->clear(); pieces->clear();
pieces->resize(m_torrent_file->num_pieces(), 1); pieces->resize(m_torrent_file->num_pieces(), 4);
return; return;
} }
@ -4936,8 +4936,8 @@ namespace libtorrent
else if (prio > 7) prio = 7; else if (prio > 7) prio = 7;
if (int(m_file_priority.size()) <= index) if (int(m_file_priority.size()) <= index)
{ {
// any unallocated slot is assumed to be 1 // any unallocated slot is assumed to be 4
if (prio == 1) return; if (prio == 4) return;
m_file_priority.resize(index + 1, 4); m_file_priority.resize(index + 1, 4);
} }
@ -5026,6 +5026,8 @@ namespace libtorrent
if (file_prio == 0) if (file_prio == 0)
{ {
// the pieces already start out as priority 0, no need to update
// the pieces vector in this case
need_update = true; need_update = true;
continue; continue;
} }
@ -5042,7 +5044,6 @@ namespace libtorrent
, pieces.begin() + last_piece + 1 , pieces.begin() + last_piece + 1
, std::bind(&set_if_greater, _1, file_prio)); , std::bind(&set_if_greater, _1, file_prio));
if (has_picker() || file_prio != 1)
need_update = true; need_update = true;
} }
if (need_update) prioritize_pieces(pieces); if (need_update) prioritize_pieces(pieces);

View File

@ -446,7 +446,9 @@ void udp_socket::bind(udp::endpoint const& ep, error_code& ec)
m_socket.io_control(ioc, ec); m_socket.io_control(ioc, ec);
if (ec) return; if (ec) return;
m_bind_port = ep.port(); error_code err;
m_bind_port = m_socket.local_endpoint(err).port();
if (err) m_bind_port = ep.port();
} }
void udp_socket::set_proxy_settings(aux::proxy_settings const& ps) void udp_socket::set_proxy_settings(aux::proxy_settings const& ps)

View File

@ -75,16 +75,16 @@ web_peer_connection::web_peer_connection(peer_connection_args const& pack
std::shared_ptr<torrent> tor = pack.tor.lock(); std::shared_ptr<torrent> tor = pack.tor.lock();
TORRENT_ASSERT(tor); TORRENT_ASSERT(tor);
// we always prefer downloading 1 MiB chunks // if the web server is known not to support keep-alive. request 4MiB
// from web seeds, or whole pieces if pieces // but we want to have at least piece size to prevent block based requests
// are larger than a MiB int const min_size = std::max((web.supports_keepalive ? 1 : 4) * 1024 * 1024,
int preferred_size = 1024 * 1024; tor->torrent_file().piece_length());
// if the web server is known not to support keep-alive. // we prefer downloading large chunks from web seeds,
// request even larger blocks at a time // but still want to be able to split requests
if (!web.supports_keepalive) preferred_size *= 4; int const preferred_size = std::max(min_size, m_settings.get_int(settings_pack::urlseed_max_request_bytes));
prefer_contiguous_blocks((std::max)(preferred_size / tor->block_size(), 1)); prefer_contiguous_blocks(preferred_size / tor->block_size());
std::shared_ptr<torrent> t = associated_torrent().lock(); std::shared_ptr<torrent> t = associated_torrent().lock();
bool const single_file_request = t->torrent_file().num_files() == 1; bool const single_file_request = t->torrent_file().num_files() == 1;