merged RC_1_1 into master

This commit is contained in:
arvidn 2017-03-14 12:55:13 -04:00
commit 3a1379ef1e
21 changed files with 275 additions and 80 deletions

View File

@ -59,6 +59,12 @@
* resume data no longer has timestamps of files
* require C++11 to build libtorrent
* document deprecation of dynamic loading/unloading of torrents
* include user-agent in tracker announces in anonymous_mode for private torrents
* add support for IPv6 peers from udp trackers
* correctly URL encode the IPv6 argument to trackers
* fix default file pool size on windows
* fix bug where settings_pack::file_pool_size setting was not being honored
* add feature to periodically close files (to make windows clear disk cache)
* fix bug in torrent_handle::file_status
* fix issue with peers not updated on metadata from magnet links

View File

@ -268,6 +268,9 @@ namespace libtorrent
bool is_paused() const;
#ifndef TORRENT_NO_DEPRECATE
// *the feature of dynamically loading/unloading torrents is deprecated
// and discouraged*
//
// This function enables dynamic-loading-of-torrent-files_. When a
// torrent is unloaded but needs to be available in memory, this function
// is called **from within the libtorrent network thread**. From within

View File

@ -802,9 +802,7 @@ namespace libtorrent
// for viruses. deferring the closing of the files will be the
// difference between a usable system and a completely hogged down
// system. Most operating systems also has a limit on the total number
// of file descriptors a process may have open. It is usually a good
// idea to find this limit and set the number of connections and the
// number of files limits so their sum is slightly below it.
// of file descriptors a process may have open.
file_pool_size,
// ``max_failcount`` is the maximum times we try to connect to a peer

View File

@ -94,6 +94,7 @@ namespace libtorrent
, kind(announce_request)
, key(0)
, num_want(0)
, private_torrent(false)
, triggered_manually(false)
#ifdef TORRENT_USE_OPENSSL
, ssl_ctx(0)
@ -152,6 +153,10 @@ namespace libtorrent
peer_id pid;
address bind_ip;
// set to true if the .torrent file this tracker announce is for is marked
// as private (i.e. has the "priv": 1 key)
bool private_torrent;
// this is set to true if this request was triggered by a "manual" call to
// scrape_tracker() or force_reannounce()
bool triggered_manually;

View File

@ -38,6 +38,7 @@ POSSIBILITY OF SUCH DAMAGE.
#include "libtorrent/session.hpp"
#include "libtorrent/session_stats.hpp"
#include "libtorrent/file.hpp"
#include "libtorrent/torrent_info.hpp"
using namespace libtorrent;
@ -83,3 +84,55 @@ TORRENT_TEST(close_file_interval)
TEST_CHECK(ran_to_completion);
}
TORRENT_TEST(file_pool_size)
{
bool ran_to_completion = false;
int max_files = 0;
setup_swarm(2, swarm_test::download
// add session
, [](lt::settings_pack& pack)
{
pack.set_int(lt::settings_pack::file_pool_size, 5);
}
// add torrent
, [](lt::add_torrent_params& atp) {
// we need a torrent with lots of files in it, to hit the
// file_size_limit we set.
file_storage fs;
for (int i = 0; i < 0x10 * 9; ++i)
{
char filename[50];
snprintf(filename, sizeof(filename), "root/file-%d", i);
fs.add_file(filename, 0x400);
}
atp.ti = std::make_shared<torrent_info>(*atp.ti);
atp.ti->remap_files(fs);
}
// on alert
, [&](lt::alert const* a, lt::session&)
{}
// terminate
, [&](int ticks, lt::session& ses) -> bool
{
if (ticks > 80)
{
TEST_ERROR("timeout");
return true;
}
std::vector<pool_file_status> status;
ses.get_torrents().at(0).file_status(status);
printf("open files: %d\n", int(status.size()));
max_files = std::max(max_files, int(status.size()));
if (!is_seed(ses)) return false;
printf("completed in %d ticks\n", ticks);
ran_to_completion = true;
return true;
});
TEST_CHECK(max_files <= 5);
TEST_CHECK(max_files >= 4);
TEST_CHECK(ran_to_completion);
}

View File

@ -755,6 +755,9 @@ TORRENT_TEST(tracker_ipv6_argument)
std::string::size_type pos = req.find("&ipv6=");
TEST_CHECK(pos != std::string::npos || stop_event);
got_ipv6 |= pos != std::string::npos;
// make sure the IPv6 argument is url encoded
TEST_CHECK(req.substr(pos + 6, req.find_first_of(pos + 6, '&'))
== "ffff%3a%3a1337");
return sim::send_response(200, "OK", 11) + "d5:peers0:e";
}
, [](torrent_handle) {}
@ -821,6 +824,61 @@ TORRENT_TEST(tracker_ipv6_argument_privacy_mode)
TEST_EQUAL(got_ipv6, false);
}
TORRENT_TEST(tracker_user_agent_privacy_mode_public_torrent)
{
bool got_announce = false;
tracker_test(
[](lt::add_torrent_params& p, lt::session& ses)
{
settings_pack pack;
pack.set_bool(settings_pack::anonymous_mode, true);
pack.set_str(settings_pack::user_agent, "test_agent/1.2.3");
ses.apply_settings(pack);
p.ti = make_torrent(false);
return 60;
},
[&](std::string method, std::string req
, std::map<std::string, std::string>& headers)
{
got_announce = true;
// in anonymous mode we should not send a user agent
TEST_CHECK(headers["user-agent"] == "");
return sim::send_response(200, "OK", 11) + "d5:peers0:e";
}
, [](torrent_handle h) {}
, [](torrent_handle h) {});
TEST_EQUAL(got_announce, true);
}
TORRENT_TEST(tracker_user_agent_privacy_mode_private_torrent)
{
bool got_announce = false;
tracker_test(
[](lt::add_torrent_params& p, lt::session& ses)
{
settings_pack pack;
pack.set_bool(settings_pack::anonymous_mode, true);
pack.set_str(settings_pack::user_agent, "test_agent/1.2.3");
ses.apply_settings(pack);
p.ti = make_torrent(true);
return 60;
},
[&](std::string method, std::string req
, std::map<std::string, std::string>& headers)
{
got_announce = true;
// in anonymous mode we should still send the user agent for private
// torrents (since private trackers sometimes require it)
TEST_CHECK(headers["user-agent"] == "test_agent/1.2.3");
return sim::send_response(200, "OK", 11) + "d5:peers0:e";
}
, [](torrent_handle h) {}
, [](torrent_handle h) {});
TEST_EQUAL(got_announce, true);
}
// TODO: test external IP
// TODO: test with different queuing settings
// TODO: test when a torrent transitions from downloading to finished and

View File

@ -55,10 +55,6 @@ POSSIBILITY OF SUCH DAMAGE.
#include <functional>
#if TORRENT_USE_RLIMIT
#include <sys/resource.h>
#endif
#define DEBUG_DISK_THREAD 0
#if DEBUG_DISK_THREAD
@ -203,15 +199,6 @@ namespace libtorrent
{
ADD_OUTSTANDING_ASYNC("disk_io_thread::work");
m_disk_cache.set_settings(m_settings);
// deduct some margin for epoll/kqueue, log files,
// futexes, shared objects etc.
// 80% of the available file descriptors should go to connections
// 20% goes towards regular files
const int max_files = std::min(std::max(5
, (max_open_files() - 20) * 2 / 10)
, m_file_pool.size_limit());
m_file_pool.resize(max_files);
}
storage_interface* disk_io_thread::get_torrent(storage_index_t const storage)
@ -296,6 +283,7 @@ namespace libtorrent
std::unique_lock<std::mutex> l(m_cache_mutex);
apply_pack(pack, m_settings);
m_disk_cache.set_settings(m_settings);
m_file_pool.resize(m_settings.get_int(settings_pack::file_pool_size));
int const num_threads = m_settings.get_int(settings_pack::aio_threads);
// add one hasher thread for every three generic threads

View File

@ -189,7 +189,7 @@ namespace libtorrent
if (!err)
{
url += "&ipv6=";
url += ip;
url += escape_string(ip);
}
}
#endif
@ -204,10 +204,16 @@ namespace libtorrent
#endif
);
int const timeout = tracker_req().event==tracker_request::stopped
int const timeout = tracker_req().event == tracker_request::stopped
? settings.get_int(settings_pack::stop_tracker_timeout)
: settings.get_int(settings_pack::tracker_completion_timeout);
// in anonymous mode we omit the user agent to mitigate fingerprinting of
// the client. Private torrents is an exception because some private
// trackers may requre the user agent
std::string const user_agent = settings.get_bool(settings_pack::anonymous_mode)
&& !tracker_req().private_torrent ? "" : settings.get_str(settings_pack::user_agent);
// when sending stopped requests, prefer the cached DNS entry
// to avoid being blocked for slow or failing responses. Chances
// are that we're shutting down, and this should be a best-effort
@ -216,9 +222,7 @@ namespace libtorrent
m_tracker_connection->get(url, seconds(timeout)
, tracker_req().event == tracker_request::stopped ? 2 : 1
, ps.proxy_tracker_connections ? &ps : nullptr
, 5, settings.get_bool(settings_pack::anonymous_mode)
? "" : settings.get_str(settings_pack::user_agent)
, bind_interface()
, 5, user_agent, bind_interface()
, tracker_req().event == tracker_request::stopped
? resolver_interface::prefer_cache
: resolver_interface::abort_on_shutdown

View File

@ -545,7 +545,7 @@ namespace libtorrent { namespace dht
#if TORRENT_USE_IPV6
: m_dht6.nid() == nid ? &m_dht6 : nullptr;
#else
: nullptr
: nullptr;
#endif
if (dht == nullptr) return ret;

View File

@ -48,13 +48,6 @@ POSSIBILITY OF SUCH DAMAGE.
#include <boost/asio/ip/v6_only.hpp>
#if TORRENT_USE_RLIMIT
#include <sys/resource.h>
// capture this here where warnings are disabled (the macro generates warnings)
const rlim_t rlim_infinity = RLIM_INFINITY;
#endif // TORRENT_USE_RLIMIT
#include "libtorrent/aux_/disable_warnings_pop.hpp"
#include "libtorrent/aux_/openssl.hpp"
@ -555,13 +548,13 @@ namespace aux {
#endif // TORRENT_DISABLE_LOGGING
// ---- auto-cap max connections ----
int max_files = max_open_files();
int const max_files = max_open_files();
// deduct some margin for epoll/kqueue, log files,
// futexes, shared objects etc.
// 80% of the available file descriptors should go to connections
m_settings.set_int(settings_pack::connections_limit, (std::min)(
m_settings.set_int(settings_pack::connections_limit, std::min(
m_settings.get_int(settings_pack::connections_limit)
, (std::max)(5, (max_files - 20) * 8 / 10)));
, std::max(5, (max_files - 20) * 8 / 10)));
// 20% goes towards regular files (see disk_io_thread)
#ifndef TORRENT_DISABLE_LOGGING
if (should_log())
@ -3620,6 +3613,7 @@ namespace aux {
else
{
t->resume();
if (!t->should_check_files()) continue;
t->start_checking();
--limit;
}
@ -6202,9 +6196,7 @@ namespace aux {
return;
}
if (m_upnp)
m_upnp->set_user_agent("");
m_settings.set_str(settings_pack::user_agent, "");
if (m_upnp) m_upnp->set_user_agent("");
url_random(m_peer_id.data(), m_peer_id.data() + 20);
}

View File

@ -252,7 +252,7 @@ namespace libtorrent
SET(active_tracker_limit, 1600, nullptr),
SET(active_lsd_limit, 60, nullptr),
SET(active_limit, 15, &session_impl::trigger_auto_manage),
DEPRECATED_SET(active_loaded_limit, 100, &session_impl::trigger_auto_manage),
DEPRECATED_SET(active_loaded_limit, 0, &session_impl::trigger_auto_manage),
SET(auto_manage_interval, 30, nullptr),
SET(seed_time_limit, 24 * 60 * 60, nullptr),
SET(auto_scrape_interval, 1800, nullptr),

View File

@ -2622,6 +2622,8 @@ namespace libtorrent
&& m_apply_ip_filter)
req.filter = m_ip_filter;
req.private_torrent = m_torrent_file->priv();
req.info_hash = m_torrent_file->info_hash();
req.pid = m_ses.get_peer_id();
req.downloaded = m_stat.total_payload_download() - m_total_failed_bytes;
@ -2829,6 +2831,7 @@ namespace libtorrent
req.info_hash = m_torrent_file->info_hash();
req.kind |= tracker_request::scrape_request;
req.url = m_trackers[idx].url;
req.private_torrent = m_torrent_file->priv();
#ifndef TORRENT_NO_DEPRECATE
req.auth = tracker_login();
#endif
@ -3120,6 +3123,9 @@ namespace libtorrent
// and they should be announced to in parallel
tracker_request req = r;
req.private_torrent = m_torrent_file->priv();
// tell the tracker to bind to the opposite protocol type
req.bind_ip = tracker_ip.is_v4()
? m_ses.get_ipv6_interface().address()

View File

@ -597,8 +597,15 @@ namespace libtorrent
resp.min_interval = seconds32(60);
resp.incomplete = aux::read_int32(buf);
resp.complete = aux::read_int32(buf);
int const num_peers = int(buf.size()) / 6;
if ((buf.size() % 6) != 0)
std::size_t const ip_stride =
#if TORRENT_USE_IPV6
m_target.address().is_v6() ? 18 :
#endif
6;
int const num_peers = static_cast<int>(buf.size() / ip_stride);
if (buf.size() % ip_stride != 0)
{
fail(error_code(errors::invalid_tracker_response_length));
return false;
@ -618,15 +625,32 @@ namespace libtorrent
return true;
}
#if TORRENT_USE_IPV6
if (m_target.address().is_v6())
{
resp.peers6.reserve(std::size_t(num_peers));
for (int i = 0; i < num_peers; ++i)
{
ipv6_peer_entry e;
std::memcpy(e.ip.data(), buf.data(), 16);
buf = buf.subspan(16);
e.port = aux::read_uint16(buf);
resp.peers6.push_back(e);
}
}
else
#endif
{
resp.peers4.reserve(std::size_t(num_peers));
for (int i = 0; i < num_peers; ++i)
{
ipv4_peer_entry e;
std::memcpy(e.ip.data(), buf.data(), 4);
memcpy(e.ip.data(), buf.data(), 4);
buf = buf.subspan(4);
e.port = aux::read_uint16(buf);
resp.peers4.push_back(e);
}
}
std::list<address> ip_list;
for (auto const& endp : m_endpoints)

View File

@ -126,23 +126,29 @@ namespace libtorrent
request += "Host: ";
request += m_host;
if ((m_first_request || m_settings.get_bool(settings_pack::always_send_user_agent))
&& !m_settings.get_bool(settings_pack::anonymous_mode)) {
&& !m_settings.get_bool(settings_pack::anonymous_mode))
{
request += "\r\nUser-Agent: ";
request += m_settings.get_str(settings_pack::user_agent);
}
if (!m_external_auth.empty()) {
if (!m_external_auth.empty())
{
request += "\r\nAuthorization: ";
request += m_external_auth;
} else if (!m_basic_auth.empty()) {
}
else if (!m_basic_auth.empty())
{
request += "\r\nAuthorization: Basic ";
request += m_basic_auth;
}
if (sett.get_int(settings_pack::proxy_type) == settings_pack::http_pw) {
if (sett.get_int(settings_pack::proxy_type) == settings_pack::http_pw)
{
request += "\r\nProxy-Authorization: Basic ";
request += base64encode(sett.get_str(settings_pack::proxy_username)
+ ":" + sett.get_str(settings_pack::proxy_password));
}
for (auto const& h : m_extra_headers) {
for (auto const& h : m_extra_headers)
{
request += "\r\n";
request += h.first;
request += ": ";

View File

@ -291,7 +291,7 @@ void save_file(char const* filename, char const* data, int size)
bool print_alerts(lt::session& ses, char const* name
, bool allow_disconnects, bool allow_no_torrents, bool allow_failed_fastresume
, bool (*predicate)(libtorrent::alert const*), bool no_output)
, std::function<bool(libtorrent::alert const*)> predicate, bool no_output)
{
bool ret = false;
std::vector<torrent_handle> handles = ses.get_torrents();

View File

@ -78,7 +78,8 @@ EXPORT bool print_alerts(libtorrent::session& ses, char const* name
, bool allow_disconnects = false
, bool allow_no_torrents = false
, bool allow_failed_fastresume = false
, bool (*)(libtorrent::alert const*) = 0
, std::function<bool(libtorrent::alert const*)> predicate
= std::function<bool(libtorrent::alert const*)>()
, bool no_output = false);
EXPORT void wait_for_listen(libtorrent::session& ses, char const* name);

View File

@ -1523,16 +1523,18 @@ void test_routing_table(address(&rand_addr)())
address node_addr;
address node_near_addr;
if (t.source.protocol() == udp::v4())
{
node_addr = addr4("4.4.4.4");
node_near_addr = addr4("4.4.4.5");
}
else
#if TORRENT_USE_IPV6
if (t.source.protocol() == udp::v6())
{
node_addr = addr6("2001:1111:1111:1111:1111:1111:1111:1111");
node_near_addr = addr6("2001:1111:1111:1111:eeee:eeee:eeee:eeee");
}
else
#endif
{
node_addr = addr4("4.4.4.4");
node_near_addr = addr4("4.4.4.5");
}
// test a node with the same IP:port changing ID
add_and_replace(tmp, diff);
@ -2500,6 +2502,7 @@ TORRENT_TEST(traversal_done)
g_put_count = 0;
}
#if TORRENT_USE_IPV6
TORRENT_TEST(dht_dual_stack)
{
// TODO: 3 use dht_test_setup class to simplify the node setup
@ -2629,6 +2632,7 @@ TORRENT_TEST(dht_dual_stack)
TEST_ERROR(error_string);
}
}
#endif
TORRENT_TEST(signing_test1)
{

View File

@ -123,6 +123,7 @@ TORRENT_TEST(announce_peer)
TEST_CHECK(!peers.find_key("values"));
}
#if TORRENT_USE_IPV6
TORRENT_TEST(dual_stack)
{
dht_settings sett = test_settings();
@ -148,6 +149,7 @@ TORRENT_TEST(dual_stack)
s->get_peers(n1, false, false, address_v6(), peers6);
TEST_EQUAL(peers6["values"].list().size(), 2);
}
#endif
TORRENT_TEST(put_items)
{

View File

@ -321,17 +321,25 @@ TORRENT_TEST(extract_peer_missing_port)
, errors::invalid_tracker_response, false);
}
TORRENT_TEST(udp_tracker)
bool connect_alert(libtorrent::alert const* a, tcp::endpoint& ep)
{
int http_port = start_web_server();
int udp_port = start_udp_tracker();
if (peer_connect_alert const* pc = alert_cast<peer_connect_alert>(a))
ep = pc->endpoint;
return true;
}
void test_udp_tracker(std::string const& iface, address tracker, tcp::endpoint const& expected_peer)
{
using namespace std::placeholders;
int const udp_port = start_udp_tracker(tracker);
int prev_udp_announces = num_udp_announces();
settings_pack pack = settings();
pack.set_bool(settings_pack::announce_to_all_trackers, true);
pack.set_bool(settings_pack::announce_to_all_tiers, true);
pack.set_str(settings_pack::listen_interfaces, "0.0.0.0:48875");
pack.set_str(settings_pack::listen_interfaces, iface + ":48875");
std::unique_ptr<lt::session> s(new lt::session(pack));
@ -343,12 +351,9 @@ TORRENT_TEST(udp_tracker)
file.close();
char tracker_url[200];
std::snprintf(tracker_url, sizeof(tracker_url), "http://127.0.0.1:%d/announce", http_port);
std::snprintf(tracker_url, sizeof(tracker_url), "udp://%s:%d/announce", iface.c_str(), udp_port);
t->add_tracker(tracker_url, 0);
std::snprintf(tracker_url, sizeof(tracker_url), "udp://127.0.0.1:%d/announce", udp_port);
t->add_tracker(tracker_url, 1);
add_torrent_params addp;
addp.flags &= ~add_torrent_params::flag_paused;
addp.flags &= ~add_torrent_params::flag_auto_managed;
@ -357,9 +362,11 @@ TORRENT_TEST(udp_tracker)
addp.save_path = "tmp1_tracker";
torrent_handle h = s->add_torrent(addp);
tcp::endpoint peer_ep;
for (int i = 0; i < 50; ++i)
{
print_alerts(*s, "s");
print_alerts(*s, "s", false, false, false, std::bind(&connect_alert, _1, std::ref(peer_ep)));
if (num_udp_announces() == prev_udp_announces + 1)
break;
@ -379,7 +386,7 @@ TORRENT_TEST(udp_tracker)
for (int i = 0; i < 50; ++i)
{
print_alerts(*s, "s", true, true);
print_alerts(*s, "s", true, true, false, std::bind(&connect_alert, _1, std::ref(peer_ep)));
if (num_udp_announces() == prev_udp_announces + 2)
break;
@ -388,7 +395,9 @@ TORRENT_TEST(udp_tracker)
, int(prev_udp_announces) + 1);
}
TEST_CHECK(peer_ep == expected_peer);
std::printf("destructing session\n");
s.reset();
std::printf("done\n");
@ -396,6 +405,18 @@ TORRENT_TEST(udp_tracker)
TEST_EQUAL(num_udp_announces(), prev_udp_announces + 2);
}
TORRENT_TEST(udp_tracker_v4)
{
test_udp_tracker("127.0.0.1", address_v4::any(), ep("1.3.3.7", 1337));
}
#if TORRENT_USE_IPV6
TORRENT_TEST(udp_tracker_v6)
{
test_udp_tracker("[::1]", address_v6::any(), ep("::1.3.3.7", 1337));
}
#endif
TORRENT_TEST(http_peers)
{
int http_port = start_web_server();

View File

@ -131,8 +131,29 @@ struct udp_tracker
detail::write_uint32(1800, ptr); // interval
detail::write_uint32(1, ptr); // incomplete
detail::write_uint32(1, ptr); // complete
// 0 peers
m_socket.send_to(boost::asio::buffer(buffer, 20), *from, 0, e);
// 1 peers
#if TORRENT_USE_IPV6
if (from->address().is_v6())
{
detail::write_uint32(0, ptr);
detail::write_uint32(0, ptr);
detail::write_uint32(0, ptr);
detail::write_uint8(1, ptr);
detail::write_uint8(3, ptr);
detail::write_uint8(3, ptr);
detail::write_uint8(7, ptr);
detail::write_uint16(1337, ptr);
}
else
#endif
{
detail::write_uint8(1, ptr);
detail::write_uint8(3, ptr);
detail::write_uint8(3, ptr);
detail::write_uint8(7, ptr);
detail::write_uint16(1337, ptr);
}
m_socket.send_to(boost::asio::buffer(buffer, ptr - buffer), *from, 0, e);
if (e) std::printf("%s: UDP send_to failed. ERROR: %s\n"
, time_now_string(), e.message().c_str());
else std::printf("%s: UDP sent response to: %s\n"
@ -153,21 +174,21 @@ struct udp_tracker
, std::bind(&udp_tracker::on_udp_receive, this, _1, _2, from, buffer, size));
}
udp_tracker()
udp_tracker(address iface)
: m_udp_announces(0)
, m_socket(m_ios)
, m_port(0)
, m_abort(false)
{
error_code ec;
m_socket.open(udp::v4(), ec);
m_socket.open(iface.is_v4() ? udp::v4() : udp::v6(), ec);
if (ec)
{
std::printf("UDP Error opening listen UDP tracker socket: %s\n", ec.message().c_str());
return;
}
m_socket.bind(udp::endpoint(address_v4::any(), 0), ec);
m_socket.bind(udp::endpoint(iface, 0), ec);
if (ec)
{
std::printf("UDP Error binding UDP tracker socket to port 0: %s\n", ec.message().c_str());
@ -187,6 +208,7 @@ struct udp_tracker
void stop()
{
std::printf("%s: UDP tracker, stop\n", time_now_string());
m_abort = true;
m_socket.cancel();
m_socket.close();
@ -233,9 +255,9 @@ struct udp_tracker
std::shared_ptr<udp_tracker> g_udp_tracker;
int start_udp_tracker()
int start_udp_tracker(address iface)
{
g_udp_tracker.reset(new udp_tracker);
g_udp_tracker.reset(new udp_tracker(iface));
return g_udp_tracker->port();
}

View File

@ -31,9 +31,11 @@ POSSIBILITY OF SUCH DAMAGE.
*/
#include "test.hpp" // for EXPORT
#include "libtorrent/address.hpp"
// returns the port the udp tracker is running on
int EXPORT start_udp_tracker();
int EXPORT start_udp_tracker(libtorrent::address iface
= libtorrent::address_v4::any());
// the number of udp tracker announces received
int EXPORT num_udp_announces();