similar to how low RTT DHT nodes are preferred over high RTT nodes, also have an affinity for nodes with node IDs derived from their IP

This commit is contained in:
arvidn 2019-01-11 01:43:45 +01:00 committed by Arvid Norberg
parent ca27892111
commit 3995ffeafd
22 changed files with 214 additions and 114 deletions

View File

@ -1,3 +1,4 @@
* add DHT routing table affinity for BEP 42 nodes
* add torrent_info constructor overloads to control torrent file limits * add torrent_info constructor overloads to control torrent file limits
* feature to disable DHT, PEX and LSD per torrent * feature to disable DHT, PEX and LSD per torrent
* fix issue where trackers from magnet links were not included in create_torrent() * fix issue where trackers from magnet links were not included in create_torrent()

View File

@ -42,7 +42,7 @@ POSSIBILITY OF SUCH DAMAGE.
using namespace lt; using namespace lt;
#if LIBTORRENT_VERSION_NUM >= 10200 #if LIBTORRENT_VERSION_NUM >= 10200
dht::dht_settings sett; dht::settings sett;
dht::dht_state state; dht::dht_state state;
std::unique_ptr<lt::dht::dht_storage_interface> dht_storage(dht::dht_default_storage_constructor(sett)); std::unique_ptr<lt::dht::dht_storage_interface> dht_storage(dht::dht_default_storage_constructor(sett));
#else #else

View File

@ -788,6 +788,7 @@ namespace aux {
void update_dht(); void update_dht();
void update_count_slow(); void update_count_slow();
void update_dht_bootstrap_nodes(); void update_dht_bootstrap_nodes();
void update_dht_settings();
void update_socket_buffer_size(); void update_socket_buffer_size();
void update_dht_announce_interval(); void update_dht_announce_interval();
@ -1088,7 +1089,7 @@ namespace aux {
#ifndef TORRENT_DISABLE_DHT #ifndef TORRENT_DISABLE_DHT
std::unique_ptr<dht::dht_storage_interface> m_dht_storage; std::unique_ptr<dht::dht_storage_interface> m_dht_storage;
std::shared_ptr<dht::dht_tracker> m_dht; std::shared_ptr<dht::dht_tracker> m_dht;
dht::dht_settings m_dht_settings; dht::settings m_dht_settings;
dht::dht_storage_constructor_type m_dht_storage_constructor dht::dht_storage_constructor_type m_dht_storage_constructor
= dht::dht_default_storage_constructor; = dht::dht_default_storage_constructor;

View File

@ -163,7 +163,15 @@ namespace dht {
// same as the tcp interface // same as the tcp interface
int service_port = 0; int service_port = 0;
#endif #endif
};
// internal
struct settings : dht_settings
{
// when this is true, nodes whose IDs are derived from their source IP
// according to BEP 42 (http://bittorrent.org/beps/bep_0042.html) are
// preferred in the routing table.
bool prefer_verified_node_ids = true;
}; };

View File

@ -56,7 +56,7 @@ namespace libtorrent {
} }
namespace libtorrent { namespace dht { namespace libtorrent { namespace dht {
struct dht_settings; struct settings;
struct TORRENT_EXTRA_EXPORT dht_tracker final struct TORRENT_EXTRA_EXPORT dht_tracker final
: socket_manager : socket_manager
@ -69,7 +69,7 @@ namespace libtorrent { namespace dht {
dht_tracker(dht_observer* observer dht_tracker(dht_observer* observer
, io_service& ios , io_service& ios
, send_fun_t const& send_fun , send_fun_t const& send_fun
, dht_settings const& settings , dht::settings const& settings
, counters& cnt , counters& cnt
, dht_storage_interface& storage , dht_storage_interface& storage
, dht_state&& state); , dht_state&& state);
@ -78,7 +78,7 @@ namespace libtorrent { namespace dht {
dht_tracker(dht_observer* observer dht_tracker(dht_observer* observer
, io_service& ios , io_service& ios
, send_fun_t const& send_fun , send_fun_t const& send_fun
, dht_settings const& settings , dht::settings const& settings
, counters& cnt , counters& cnt
, dht_storage_interface& storage , dht_storage_interface& storage
, dht_state const& state) = delete; , dht_state const& state) = delete;
@ -159,7 +159,7 @@ namespace libtorrent { namespace dht {
{ {
tracker_node(io_service& ios tracker_node(io_service& ios
, aux::listen_socket_handle const& s, socket_manager* sock , aux::listen_socket_handle const& s, socket_manager* sock
, dht_settings const& settings , dht::settings const& settings
, node_id const& nid , node_id const& nid
, dht_observer* observer, counters& cnt , dht_observer* observer, counters& cnt
, get_foreign_node_t get_foreign_node , get_foreign_node_t get_foreign_node
@ -202,7 +202,7 @@ namespace libtorrent { namespace dht {
deadline_timer m_key_refresh_timer; deadline_timer m_key_refresh_timer;
deadline_timer m_refresh_timer; deadline_timer m_refresh_timer;
dht_settings const& m_settings; dht::settings const& m_settings;
bool m_running; bool m_running;

View File

@ -91,7 +91,7 @@ class TORRENT_EXTRA_EXPORT node
{ {
public: public:
node(aux::listen_socket_handle const& sock, socket_manager* sock_man node(aux::listen_socket_handle const& sock, socket_manager* sock_man
, dht_settings const& settings , dht::settings const& settings
, node_id const& nid , node_id const& nid
, dht_observer* observer, counters& cnt , dht_observer* observer, counters& cnt
, get_foreign_node_t get_foreign_node , get_foreign_node_t get_foreign_node
@ -198,7 +198,7 @@ public:
void status(libtorrent::session_status& s); void status(libtorrent::session_status& s);
#endif #endif
dht_settings const& settings() const { return m_settings; } dht::settings const& settings() const { return m_settings; }
counters& stats_counters() const { return m_counters; } counters& stats_counters() const { return m_counters; }
dht_observer* observer() const { return m_observer; } dht_observer* observer() const { return m_observer; }
@ -224,7 +224,7 @@ private:
bool lookup_peers(sha1_hash const& info_hash, entry& reply bool lookup_peers(sha1_hash const& info_hash, entry& reply
, bool noseed, bool scrape, address const& requester) const; , bool noseed, bool scrape, address const& requester) const;
dht_settings const& m_settings; dht::settings const& m_settings;
std::mutex m_mutex; std::mutex m_mutex;

View File

@ -38,6 +38,7 @@ POSSIBILITY OF SUCH DAMAGE.
#include "libtorrent/address.hpp" #include "libtorrent/address.hpp"
#include "libtorrent/union_endpoint.hpp" #include "libtorrent/union_endpoint.hpp"
#include "libtorrent/time.hpp" // for time_point #include "libtorrent/time.hpp" // for time_point
#include "libtorrent/aux_/time.hpp" // for time_now
namespace libtorrent { namespace dht { namespace libtorrent { namespace dht {
@ -46,7 +47,7 @@ struct TORRENT_EXTRA_EXPORT node_entry
node_entry(node_id const& id_, udp::endpoint const& ep, int roundtriptime = 0xffff node_entry(node_id const& id_, udp::endpoint const& ep, int roundtriptime = 0xffff
, bool pinged = false); , bool pinged = false);
explicit node_entry(udp::endpoint const& ep); explicit node_entry(udp::endpoint const& ep);
node_entry(); node_entry() = default;
void update_rtt(int new_rtt); void update_rtt(int new_rtt);
bool pinged() const { return timeout_count != 0xff; } bool pinged() const { return timeout_count != 0xff; }
@ -59,23 +60,32 @@ struct TORRENT_EXTRA_EXPORT node_entry
address addr() const { return endpoint.address(); } address addr() const { return endpoint.address(); }
int port() const { return endpoint.port; } int port() const { return endpoint.port; }
// compares which node_entry is "better". Smaller is better
bool operator<(node_entry const& rhs) const
{
return std::make_tuple(!verified, rtt) < std::make_tuple(!rhs.verified, rhs.rtt);
}
#ifndef TORRENT_DISABLE_LOGGING #ifndef TORRENT_DISABLE_LOGGING
time_point first_seen; time_point first_seen = aux::time_now();
#endif #endif
// the time we last received a response for a request to this peer // the time we last received a response for a request to this peer
time_point last_queried; time_point last_queried = min_time();
node_id id; node_id id{nullptr};
union_endpoint endpoint; union_endpoint endpoint;
// the average RTT of this node // the average RTT of this node
std::uint16_t rtt; std::uint16_t rtt = 0xffff;
// the number of times this node has failed to // the number of times this node has failed to
// respond in a row // respond in a row
std::uint8_t timeout_count; // 0xff is a special value to indicate we have not pinged this node yet
std::uint8_t timeout_count = 0xff;
bool verified = false;
}; };
} } // namespace libtorrent::dht } } // namespace libtorrent::dht

View File

@ -49,7 +49,7 @@ POSSIBILITY OF SUCH DAMAGE.
namespace libtorrent { namespace dht { namespace libtorrent { namespace dht {
struct dht_settings; struct settings;
struct dht_logger; struct dht_logger;
using bucket_t = aux::vector<node_entry>; using bucket_t = aux::vector<node_entry>;
@ -112,7 +112,13 @@ struct ip_set
// the most times is replaced. If none of the nodes in the // the most times is replaced. If none of the nodes in the
// bucket has failed, then it is put in the replacement // bucket has failed, then it is put in the replacement
// cache (just like in the paper). // cache (just like in the paper).
// * The routing table bucket sizes are larger towards the "top" of the routing
// table. This is to get closer to the target in fewer round-trips.
// * Nodes with lower RTT are preferred and may replace nodes with higher RTT
// * Nodes that are "verified" (i.e. use a node-ID derived from their IP) are
// preferred and may replace nodes that are not verified.
TORRENT_EXTRA_EXPORT bool mostly_verified_nodes(bucket_t const&);
TORRENT_EXTRA_EXPORT bool compare_ip_cidr(address const& lhs, address const& rhs); TORRENT_EXTRA_EXPORT bool compare_ip_cidr(address const& lhs, address const& rhs);
class TORRENT_EXTRA_EXPORT routing_table class TORRENT_EXTRA_EXPORT routing_table
@ -125,7 +131,7 @@ public:
routing_table(node_id const& id, udp proto routing_table(node_id const& id, udp proto
, int bucket_size , int bucket_size
, dht_settings const& settings , dht::settings const& settings
, dht_logger* log); , dht_logger* log);
routing_table(routing_table const&) = delete; routing_table(routing_table const&) = delete;
@ -266,7 +272,7 @@ private:
void prune_empty_bucket(); void prune_empty_bucket();
dht_settings const& m_settings; dht::settings const& m_settings;
// (k-bucket, replacement cache) pairs // (k-bucket, replacement cache) pairs
// the first entry is the bucket the furthest // the first entry is the bucket the furthest

View File

@ -729,6 +729,11 @@ namespace libtorrent {
// changes are taken in consideration. // changes are taken in consideration.
enable_ip_notifier, enable_ip_notifier,
// when this is true, nodes whose IDs are derived from their source IP
// according to BEP 42 (http://bittorrent.org/beps/bep_0042.html) are
// preferred in the routing table.
dht_prefer_verified_node_ids,
max_bool_setting_internal max_bool_setting_internal
}; };

View File

@ -56,8 +56,6 @@ using namespace lt;
namespace { namespace {
lt::time_point start_time;
// this is the IP address assigned to node 'idx' // this is the IP address assigned to node 'idx'
asio::ip::address addr_from_int(int /* idx */) asio::ip::address addr_from_int(int /* idx */)
{ {
@ -90,7 +88,7 @@ namespace {
struct dht_node final : lt::dht::socket_manager struct dht_node final : lt::dht::socket_manager
{ {
dht_node(sim::simulation& sim, lt::dht::dht_settings const& sett, lt::counters& cnt dht_node(sim::simulation& sim, lt::dht::settings const& sett, lt::counters& cnt
, int const idx, std::uint32_t const flags) , int const idx, std::uint32_t const flags)
: m_io_service(sim, (flags & dht_network::bind_ipv6) ? addr6_from_int(idx) : addr_from_int(idx)) : m_io_service(sim, (flags & dht_network::bind_ipv6) ? addr6_from_int(idx) : addr_from_int(idx))
, m_dht_storage(lt::dht::dht_default_storage_constructor(sett)) , m_dht_storage(lt::dht::dht_default_storage_constructor(sett))

View File

@ -66,7 +66,7 @@ private:
// used for all the nodes in the network // used for all the nodes in the network
lt::counters m_cnt; lt::counters m_cnt;
lt::dht::dht_settings m_sett; lt::dht::settings m_sett;
std::list<dht_node> m_nodes; std::list<dht_node> m_nodes;
}; };

View File

@ -44,7 +44,6 @@ POSSIBILITY OF SUCH DAMAGE.
#include "libtorrent/deadline_timer.hpp" #include "libtorrent/deadline_timer.hpp"
#include "libtorrent/socket_io.hpp" #include "libtorrent/socket_io.hpp"
#include "setup_swarm.hpp" #include "setup_swarm.hpp"
#include "setup_dht.hpp"
#include "libtorrent/kademlia/ed25519.hpp" #include "libtorrent/kademlia/ed25519.hpp"
#include "libtorrent/bencode.hpp" #include "libtorrent/bencode.hpp"
#include "libtorrent/kademlia/item.hpp" #include "libtorrent/kademlia/item.hpp"

View File

@ -112,7 +112,7 @@ TORRENT_TEST(dht_rate_limit)
ls->local_endpoint = tcp::endpoint(address_v4::from_string("40.30.20.10"), 8888); ls->local_endpoint = tcp::endpoint(address_v4::from_string("40.30.20.10"), 8888);
error_code ec; error_code ec;
sock.bind(udp::endpoint(address_v4::from_string("40.30.20.10"), 8888), ec); sock.bind(udp::endpoint(address_v4::from_string("40.30.20.10"), 8888), ec);
dht::dht_settings dhtsett; dht::settings dhtsett;
dhtsett.block_ratelimit = 100000; // disable the DOS blocker dhtsett.block_ratelimit = 100000; // disable the DOS blocker
dhtsett.ignore_dark_internet = false; dhtsett.ignore_dark_internet = false;
dhtsett.upload_rate_limit = 400; dhtsett.upload_rate_limit = 400;
@ -239,7 +239,7 @@ TORRENT_TEST(dht_delete_socket)
ls->external_address.cast_vote(address_v4::from_string("40.30.20.10") ls->external_address.cast_vote(address_v4::from_string("40.30.20.10")
, lt::aux::session_interface::source_dht, lt::address()); , lt::aux::session_interface::source_dht, lt::address());
ls->local_endpoint = tcp::endpoint(address_v4::from_string("40.30.20.10"), 8888); ls->local_endpoint = tcp::endpoint(address_v4::from_string("40.30.20.10"), 8888);
dht::dht_settings dhtsett; dht::settings dhtsett;
counters cnt; counters cnt;
dht::dht_state state; dht::dht_state state;
std::unique_ptr<lt::dht::dht_storage_interface> dht_storage(dht::dht_default_storage_constructor(dhtsett)); std::unique_ptr<lt::dht::dht_storage_interface> dht_storage(dht::dht_default_storage_constructor(dhtsett));

View File

@ -85,7 +85,7 @@ namespace libtorrent { namespace dht {
dht_tracker::dht_tracker(dht_observer* observer dht_tracker::dht_tracker(dht_observer* observer
, io_service& ios , io_service& ios
, send_fun_t const& send_fun , send_fun_t const& send_fun
, dht_settings const& settings , dht::settings const& settings
, counters& cnt , counters& cnt
, dht_storage_interface& storage , dht_storage_interface& storage
, dht_state&& state) , dht_state&& state)
@ -576,7 +576,7 @@ namespace libtorrent { namespace dht {
dht_tracker::tracker_node::tracker_node(io_service& ios dht_tracker::tracker_node::tracker_node(io_service& ios
, aux::listen_socket_handle const& s, socket_manager* sock , aux::listen_socket_handle const& s, socket_manager* sock
, dht_settings const& settings , dht::settings const& settings
, node_id const& nid , node_id const& nid
, dht_observer* observer, counters& cnt , dht_observer* observer, counters& cnt
, get_foreign_node_t get_foreign_node , get_foreign_node_t get_foreign_node

View File

@ -106,7 +106,7 @@ void incoming_error(entry& e, char const* msg, int error_code = 203)
} // anonymous namespace } // anonymous namespace
node::node(aux::listen_socket_handle const& sock, socket_manager* sock_man node::node(aux::listen_socket_handle const& sock, socket_manager* sock_man
, dht_settings const& settings , dht::settings const& settings
, node_id const& nid , node_id const& nid
, dht_observer* observer , dht_observer* observer
, counters& cnt , counters& cnt

View File

@ -43,34 +43,13 @@ namespace libtorrent { namespace dht {
, endpoint(ep) , endpoint(ep)
, rtt(roundtriptime & 0xffff) , rtt(roundtriptime & 0xffff)
, timeout_count(pinged ? 0 : 0xff) , timeout_count(pinged ? 0 : 0xff)
, verified(verify_id(id_, ep.address()))
{ {
#ifndef TORRENT_DISABLE_LOGGING
first_seen = aux::time_now();
#endif
} }
node_entry::node_entry(udp::endpoint const& ep) node_entry::node_entry(udp::endpoint const& ep)
: last_queried(min_time()) : endpoint(ep)
, id(nullptr) {}
, endpoint(ep)
, rtt(0xffff)
, timeout_count(0xff)
{
#ifndef TORRENT_DISABLE_LOGGING
first_seen = aux::time_now();
#endif
}
node_entry::node_entry()
: last_queried(min_time())
, id(nullptr)
, rtt(0xffff)
, timeout_count(0xff)
{
#ifndef TORRENT_DISABLE_LOGGING
first_seen = aux::time_now();
#endif
}
void node_entry::update_rtt(int const new_rtt) void node_entry::update_rtt(int const new_rtt)
{ {

View File

@ -68,7 +68,7 @@ namespace {
container.erase(i); container.erase(i);
} }
bool verify_node_address(dht_settings const& settings bool verify_node_address(dht::settings const& settings
, node_id const& id, address const& addr) , node_id const& id, address const& addr)
{ {
// only when the node_id pass the verification, add it to routing table. // only when the node_id pass the verification, add it to routing table.
@ -100,8 +100,16 @@ void ip_set::erase(address const& addr)
erase_one(m_ip4s, addr.to_v4().to_bytes()); erase_one(m_ip4s, addr.to_v4().to_bytes());
} }
bool mostly_verified_nodes(bucket_t const& b)
{
int const num_verified = static_cast<int>(std::count_if(b.begin(), b.end()
, [](node_entry const& e) { return e.verified; }));
if (num_verified == 0 && b.size() > 0) return false;
return num_verified >= static_cast<int>(b.size()) * 2 / 3;
}
routing_table::routing_table(node_id const& id, udp proto, int bucket_size routing_table::routing_table(node_id const& id, udp proto, int bucket_size
, dht_settings const& settings , dht::settings const& settings
, dht_logger* log) , dht_logger* log)
: :
#ifndef TORRENT_DISABLE_LOGGING #ifndef TORRENT_DISABLE_LOGGING
@ -654,6 +662,8 @@ ip_ok:
// split the last bucket // split the last bucket
bool const can_split = (std::next(i) == m_buckets.end() bool const can_split = (std::next(i) == m_buckets.end()
&& m_buckets.size() < 159) && m_buckets.size() < 159)
&& (m_settings.prefer_verified_node_ids == false
|| (e.verified && mostly_verified_nodes(b)))
&& e.confirmed() && e.confirmed()
&& (i == m_buckets.begin() || std::prev(i)->live_nodes.size() > 1); && (i == m_buckets.begin() || std::prev(i)->live_nodes.size() > 1);
@ -742,7 +752,7 @@ ip_ok:
{ {
j = *std::max_element(nodes.begin(), nodes.end() j = *std::max_element(nodes.begin(), nodes.end()
, [](bucket_t::iterator lhs, bucket_t::iterator rhs) , [](bucket_t::iterator lhs, bucket_t::iterator rhs)
{ return lhs->rtt < rhs->rtt; }); { return *lhs < *rhs; });
} }
else else
{ {
@ -788,7 +798,7 @@ ip_ok:
auto k = std::max_element(nodes.begin(), nodes.end() auto k = std::max_element(nodes.begin(), nodes.end()
, [](bucket_t::iterator lhs, bucket_t::iterator rhs) , [](bucket_t::iterator lhs, bucket_t::iterator rhs)
{ return lhs->rtt < rhs->rtt; }); { return *lhs < *rhs; });
// in this case, we would really rather replace the node even if // in this case, we would really rather replace the node even if
// the new node has higher RTT, because it fills a new prefix that we otherwise // the new node has higher RTT, because it fills a new prefix that we otherwise
@ -798,24 +808,24 @@ ip_ok:
} }
else else
{ {
j = std::max_element(b.begin(), b.end() j = std::max_element(b.begin(), b.end());
, [](node_entry const& lhs, node_entry const& rhs)
{ return lhs.rtt < rhs.rtt; });
} }
} }
if (j != b.end() && (force_replace || j->rtt > e.rtt)) if (j != b.end() && (force_replace || e < *j))
{ {
m_ips.erase(j->addr());
*j = e;
m_ips.insert(e.addr());
#ifndef TORRENT_DISABLE_LOGGING #ifndef TORRENT_DISABLE_LOGGING
if (m_log != nullptr && m_log->should_log(dht_logger::routing_table)) if (m_log != nullptr && m_log->should_log(dht_logger::routing_table))
{ {
m_log->log(dht_logger::routing_table, "replacing node with higher RTT: %s %s" m_log->log(dht_logger::routing_table, "replacing node with better one: %s %s %s %dms vs. %s %dms"
, aux::to_hex(e.id).c_str(), print_address(e.addr()).c_str()); , aux::to_hex(e.id).c_str(), print_address(e.addr()).c_str()
, e.verified ? "verified" : "not-verified", e.rtt
, j->verified ? "verified" : "not-verified", j->rtt);
} }
#endif #endif
m_ips.erase(j->addr());
*j = e;
m_ips.insert(e.addr());
return node_added; return node_added;
} }
// in order to keep lookup times small, prefer nodes with low RTTs // in order to keep lookup times small, prefer nodes with low RTTs
@ -1056,7 +1066,7 @@ void routing_table::heard_about(node_id const& id, udp::endpoint const& ep)
// top of its bucket. the return value indicates if the table needs a refresh. // top of its bucket. the return value indicates if the table needs a refresh.
// if true, the node should refresh the table (i.e. do a find_node on its own // if true, the node should refresh the table (i.e. do a find_node on its own
// id) // id)
bool routing_table::node_seen(node_id const& id, udp::endpoint const& ep, int rtt) bool routing_table::node_seen(node_id const& id, udp::endpoint const& ep, int const rtt)
{ {
return verify_node_address(m_settings, id, ep.address()) && add_node(node_entry(id, ep, rtt, true)); return verify_node_address(m_settings, id, ep.address()) && add_node(node_entry(id, ep, rtt, true));
} }

View File

@ -648,7 +648,7 @@ namespace aux {
settings = e->dict_find_dict("dht"); settings = e->dict_find_dict("dht");
if (settings) if (settings)
{ {
m_dht_settings = dht::read_dht_settings(settings); static_cast<dht::dht_settings&>(m_dht_settings) = dht::read_dht_settings(settings);
} }
} }
@ -5346,6 +5346,16 @@ namespace aux {
#endif #endif
} }
void session_impl::update_dht_settings()
{
#ifndef TORRENT_DISABLE_DHT
bool const prefer_verified_nodes = m_settings.get_bool(
settings_pack::dht_prefer_verified_node_ids);
m_dht_settings.prefer_verified_node_ids = prefer_verified_nodes;
#endif
}
void session_impl::update_count_slow() void session_impl::update_count_slow()
{ {
error_code ec; error_code ec;
@ -5817,7 +5827,7 @@ namespace aux {
void session_impl::set_dht_settings(dht::dht_settings const& settings) void session_impl::set_dht_settings(dht::dht_settings const& settings)
{ {
m_dht_settings = settings; static_cast<dht::dht_settings&>(m_dht_settings) = settings;
} }
void session_impl::set_dht_state(dht::dht_state&& state) void session_impl::set_dht_state(dht::dht_state&& state)

View File

@ -207,6 +207,7 @@ constexpr int CLOSE_FILE_INTERVAL = 0;
SET(auto_sequential, true, &session_impl::update_auto_sequential), SET(auto_sequential, true, &session_impl::update_auto_sequential),
SET(proxy_tracker_connections, true, nullptr), SET(proxy_tracker_connections, true, nullptr),
SET(enable_ip_notifier, true, &session_impl::update_ip_notifier), SET(enable_ip_notifier, true, &session_impl::update_ip_notifier),
SET(dht_prefer_verified_node_ids, true, &session_impl::update_dht_settings),
}}); }});
aux::array<int_setting_entry_t, settings_pack::num_int_settings> const int_settings aux::array<int_setting_entry_t, settings_pack::num_int_settings> const int_settings

View File

@ -50,7 +50,7 @@ void report_failure(char const* err, char const* file, int line)
{ {
char buf[500]; char buf[500];
std::snprintf(buf, sizeof(buf), "\x1b[41m***** %s:%d \"%s\" *****\x1b[0m\n", file, line, err); std::snprintf(buf, sizeof(buf), "\x1b[41m***** %s:%d \"%s\" *****\x1b[0m\n", file, line, err);
std::fprintf(stderr, "\n%s\n", buf); std::printf("\n%s\n", buf);
failure_strings.push_back(buf); failure_strings.push_back(buf);
++_g_test_failures; ++_g_test_failures;
} }

View File

@ -563,9 +563,9 @@ struct obs : dht::dht_observer
#endif #endif
}; };
dht::dht_settings test_settings() dht::settings test_settings()
{ {
dht::dht_settings sett; dht::settings sett;
sett.max_torrents = 4; sett.max_torrents = 4;
sett.max_dht_items = 4; sett.max_dht_items = 4;
sett.enforce_node_id = false; sett.enforce_node_id = false;
@ -585,7 +585,7 @@ struct dht_test_setup
dht_storage->update_node_ids({node_id::min()}); dht_storage->update_node_ids({node_id::min()});
} }
dht::dht_settings sett; dht::settings sett;
mock_socket s; mock_socket s;
std::shared_ptr<aux::listen_socket_t> ls; std::shared_ptr<aux::listen_socket_t> ls;
obs observer; obs observer;
@ -1585,18 +1585,14 @@ void test_routing_table(address(&rand_addr)())
bdecode_node response; bdecode_node response;
// test kademlia routing table // test kademlia routing table
dht::dht_settings s; dht::settings s;
s.extended_routing_table = false; s.extended_routing_table = false;
// s.restrict_routing_ips = false; // s.restrict_routing_ips = false;
node_id id = to_hash("3123456789abcdef01232456789abcdef0123456"); node_id const nid = to_hash("3123456789abcdef01232456789abcdef0123456");
const int bucket_size = 10; const int bucket_size = 10;
dht::routing_table table(id, t.source.protocol(), bucket_size, s, &t.observer); dht::routing_table table(nid, t.source.protocol(), bucket_size, s, &t.observer);
std::vector<node_entry> nodes;
TEST_EQUAL(std::get<0>(table.size()), 0); TEST_EQUAL(std::get<0>(table.size()), 0);
node_id tmp = id;
node_id diff = to_hash("15764f7459456a9453f8719b09547c11d5f34061");
address node_addr; address node_addr;
address node_near_addr; address node_near_addr;
if (is_v6(t.source)) if (is_v6(t.source))
@ -1611,9 +1607,11 @@ void test_routing_table(address(&rand_addr)())
} }
// test a node with the same IP:port changing ID // test a node with the same IP:port changing ID
add_and_replace(tmp, diff); node_id const tmp = generate_id_impl(node_addr, 1);
table.node_seen(tmp, udp::endpoint(node_addr, 4), 10); table.node_seen(tmp, udp::endpoint(node_addr, 4), 10);
table.find_node(id, nodes, 0, 10);
std::vector<node_entry> nodes;
table.find_node(nid, nodes, 0, 10);
TEST_EQUAL(table.bucket_size(0), 1); TEST_EQUAL(table.bucket_size(0), 1);
TEST_EQUAL(std::get<0>(table.size()), 1); TEST_EQUAL(std::get<0>(table.size()), 1);
TEST_EQUAL(nodes.size(), 1); TEST_EQUAL(nodes.size(), 1);
@ -1654,7 +1652,7 @@ void test_routing_table(address(&rand_addr)())
// test adding the same node ID again with a different IP (should be ignored) // test adding the same node ID again with a different IP (should be ignored)
table.node_seen(tmp, udp::endpoint(node_addr, 5), 10); table.node_seen(tmp, udp::endpoint(node_addr, 5), 10);
table.find_node(id, nodes, 0, 10); table.find_node(nid, nodes, 0, 10);
TEST_EQUAL(table.bucket_size(0), 1); TEST_EQUAL(table.bucket_size(0), 1);
if (!nodes.empty()) if (!nodes.empty())
{ {
@ -1667,7 +1665,7 @@ void test_routing_table(address(&rand_addr)())
// very close to the current one (should be ignored) // very close to the current one (should be ignored)
// if restrict_routing_ips == true // if restrict_routing_ips == true
table.node_seen(tmp, udp::endpoint(node_near_addr, 5), 10); table.node_seen(tmp, udp::endpoint(node_near_addr, 5), 10);
table.find_node(id, nodes, 0, 10); table.find_node(nid, nodes, 0, 10);
TEST_EQUAL(table.bucket_size(0), 1); TEST_EQUAL(table.bucket_size(0), 1);
if (!nodes.empty()) if (!nodes.empty())
{ {
@ -1677,9 +1675,11 @@ void test_routing_table(address(&rand_addr)())
} }
// test adding the same IP:port again with a new node ID (should remove the node) // test adding the same IP:port again with a new node ID (should remove the node)
add_and_replace(tmp, diff); {
table.node_seen(tmp, udp::endpoint(node_addr, 4), 10); auto const id = generate_id_impl(node_addr, 2);
table.node_seen(id, udp::endpoint(node_addr, 4), 10);
table.find_node(id, nodes, 0, 10); table.find_node(id, nodes, 0, 10);
}
TEST_EQUAL(table.bucket_size(0), 0); TEST_EQUAL(table.bucket_size(0), 0);
TEST_EQUAL(nodes.size(), 0); TEST_EQUAL(nodes.size(), 0);
@ -1687,17 +1687,22 @@ void test_routing_table(address(&rand_addr)())
init_rand_address(); init_rand_address();
add_and_replace(tmp, diff); {
table.node_seen(id, rand_udp_ep(rand_addr), 10); auto const ep = rand_udp_ep(rand_addr);
auto const id = generate_id(ep.address());
table.node_seen(id, ep, 10);
}
nodes.clear(); nodes.clear();
for (int i = 0; i < 7000; ++i) for (int i = 0; i < 7000; ++i)
{ {
table.node_seen(tmp, rand_udp_ep(rand_addr), 20 + (tmp[19] & 0xff)); auto const ep = rand_udp_ep(rand_addr);
add_and_replace(tmp, diff); auto const id = generate_id(ep.address());
table.node_seen(id, ep, 20 + (id[19] & 0xff));
} }
std::printf("active buckets: %d\n", table.num_active_buckets()); std::printf("active buckets: %d\n", table.num_active_buckets());
TEST_EQUAL(table.num_active_buckets(), 10); TEST_CHECK(table.num_active_buckets() == 10
|| table.num_active_buckets() == 11);
TEST_CHECK(std::get<0>(table.size()) >= 10 * 10); TEST_CHECK(std::get<0>(table.size()) >= 10 * 10);
//TODO: 2 test num_global_nodes //TODO: 2 test num_global_nodes
//TODO: 2 test need_refresh //TODO: 2 test need_refresh
@ -1710,10 +1715,12 @@ void test_routing_table(address(&rand_addr)())
std::vector<node_entry> temp; std::vector<node_entry> temp;
aux::random_bytes(tmp); {
table.find_node(tmp, temp, 0, int(nodes.size()) * 2); node_id const id = generate_random_id();
table.find_node(id, temp, 0, int(nodes.size()) * 2);
std::printf("returned-all: %d\n", int(temp.size())); std::printf("returned-all: %d\n", int(temp.size()));
TEST_EQUAL(temp.size(), nodes.size()); TEST_EQUAL(temp.size(), nodes.size());
}
// This makes sure enough of the nodes returned are actually // This makes sure enough of the nodes returned are actually
// part of the closest nodes // part of the closest nodes
@ -1723,21 +1730,19 @@ void test_routing_table(address(&rand_addr)())
for (int r = 0; r < reps; ++r) for (int r = 0; r < reps; ++r)
{ {
aux::random_bytes(tmp); node_id const id = generate_random_id();
table.find_node(tmp, temp, 0, bucket_size * 2); table.find_node(id, temp, 0, bucket_size * 2);
std::printf("returned: %d\n", int(temp.size()));
TEST_EQUAL(int(temp.size()), std::min(bucket_size * 2, int(nodes.size()))); TEST_EQUAL(int(temp.size()), std::min(bucket_size * 2, int(nodes.size())));
std::sort(nodes.begin(), nodes.end(), std::bind(&compare_ref std::sort(nodes.begin(), nodes.end(), std::bind(&compare_ref
, std::bind(&node_entry::id, _1) , std::bind(&node_entry::id, _1)
, std::bind(&node_entry::id, _2), tmp)); , std::bind(&node_entry::id, _2), id));
int expected = std::accumulate(nodes.begin(), nodes.begin() + (bucket_size * 2) int expected = std::accumulate(nodes.begin(), nodes.begin() + (bucket_size * 2)
, 0, std::bind(&sum_distance_exp, _1, _2, tmp)); , 0, std::bind(&sum_distance_exp, _1, _2, id));
int sum_hits = std::accumulate(temp.begin(), temp.end() int sum_hits = std::accumulate(temp.begin(), temp.end()
, 0, std::bind(&sum_distance_exp, _1, _2, tmp)); , 0, std::bind(&sum_distance_exp, _1, _2, id));
TEST_EQUAL(bucket_size * 2, int(temp.size())); TEST_EQUAL(bucket_size * 2, int(temp.size()));
std::printf("expected: %d actual: %d\n", expected, sum_hits);
TEST_EQUAL(expected, sum_hits); TEST_EQUAL(expected, sum_hits);
duplicates.clear(); duplicates.clear();
@ -2667,7 +2672,7 @@ TORRENT_TEST(traversal_done)
TORRENT_TEST(dht_dual_stack) TORRENT_TEST(dht_dual_stack)
{ {
// TODO: 3 use dht_test_setup class to simplify the node setup // TODO: 3 use dht_test_setup class to simplify the node setup
dht::dht_settings sett = test_settings(); dht::settings sett = test_settings();
mock_socket s; mock_socket s;
auto sock4 = dummy_listen_socket4(); auto sock4 = dummy_listen_socket4();
auto sock6 = dummy_listen_socket6(); auto sock6 = dummy_listen_socket6();
@ -2989,10 +2994,13 @@ TORRENT_TEST(verify_message)
TORRENT_TEST(routing_table_uniform) TORRENT_TEST(routing_table_uniform)
{ {
// test routing table // test routing table
dht::dht_settings sett = test_settings(); dht::settings sett = test_settings();
obs observer; obs observer;
sett.extended_routing_table = false; sett.extended_routing_table = false;
// it's difficult to generate valid nodes with specific node IDs, so just
// turn off that check
sett.prefer_verified_node_ids = false;
node_id id = to_hash("1234876923549721020394873245098347598635"); node_id id = to_hash("1234876923549721020394873245098347598635");
node_id diff = to_hash("15764f7459456a9453f8719b09547c11d5f34061"); node_id diff = to_hash("15764f7459456a9453f8719b09547c11d5f34061");
@ -3031,10 +3039,11 @@ TORRENT_TEST(routing_table_uniform)
TORRENT_TEST(routing_table_balance) TORRENT_TEST(routing_table_balance)
{ {
dht::dht_settings sett = test_settings(); dht::settings sett = test_settings();
obs observer; obs observer;
sett.extended_routing_table = false; sett.extended_routing_table = false;
sett.prefer_verified_node_ids = false;
node_id id = to_hash("1234876923549721020394873245098347598635"); node_id id = to_hash("1234876923549721020394873245098347598635");
routing_table tbl(id, udp::v4(), 8, sett, &observer); routing_table tbl(id, udp::v4(), 8, sett, &observer);
@ -3054,9 +3063,10 @@ TORRENT_TEST(routing_table_balance)
TORRENT_TEST(routing_table_extended) TORRENT_TEST(routing_table_extended)
{ {
dht::dht_settings sett = test_settings(); dht::settings sett = test_settings();
obs observer; obs observer;
sett.extended_routing_table = true; sett.extended_routing_table = true;
sett.prefer_verified_node_ids = false;
node_id id = to_hash("1234876923549721020394873245098347598635"); node_id id = to_hash("1234876923549721020394873245098347598635");
node_id diff = to_hash("15764f7459456a9453f8719b09547c11d5f34061"); node_id diff = to_hash("15764f7459456a9453f8719b09547c11d5f34061");
@ -3088,9 +3098,10 @@ void inserter(std::set<node_id>* nodes, node_entry const& ne)
TORRENT_TEST(routing_table_set_id) TORRENT_TEST(routing_table_set_id)
{ {
dht::dht_settings sett = test_settings(); dht::settings sett = test_settings();
sett.enforce_node_id = false; sett.enforce_node_id = false;
sett.extended_routing_table = false; sett.extended_routing_table = false;
sett.prefer_verified_node_ids = false;
obs observer; obs observer;
node_id id = to_hash("0000000000000000000000000000000000000000"); node_id id = to_hash("0000000000000000000000000000000000000000");
@ -3134,10 +3145,11 @@ TORRENT_TEST(routing_table_set_id)
TORRENT_TEST(routing_table_for_each) TORRENT_TEST(routing_table_for_each)
{ {
dht::dht_settings sett = test_settings(); dht::settings sett = test_settings();
obs observer; obs observer;
sett.extended_routing_table = false; sett.extended_routing_table = false;
sett.prefer_verified_node_ids = false;
node_id id = to_hash("1234876923549721020394873245098347598635"); node_id id = to_hash("1234876923549721020394873245098347598635");
routing_table tbl(id, udp::v4(), 2, sett, &observer); routing_table tbl(id, udp::v4(), 2, sett, &observer);
@ -3205,7 +3217,7 @@ TORRENT_TEST(node_set_id)
TORRENT_TEST(read_only_node) TORRENT_TEST(read_only_node)
{ {
// TODO: 3 use dht_test_setup class to simplify the node setup // TODO: 3 use dht_test_setup class to simplify the node setup
dht::dht_settings sett = test_settings(); dht::settings sett = test_settings();
sett.read_only = true; sett.read_only = true;
mock_socket s; mock_socket s;
auto ls = dummy_listen_socket4(); auto ls = dummy_listen_socket4();
@ -3304,7 +3316,7 @@ TORRENT_TEST(read_only_node)
TORRENT_TEST(invalid_error_msg) TORRENT_TEST(invalid_error_msg)
{ {
// TODO: 3 use dht_test_setup class to simplify the node setup // TODO: 3 use dht_test_setup class to simplify the node setup
dht::dht_settings sett = test_settings(); dht::settings sett = test_settings();
mock_socket s; mock_socket s;
auto ls = dummy_listen_socket4(); auto ls = dummy_listen_socket4();
obs observer; obs observer;
@ -3357,6 +3369,8 @@ struct test_algo : dht::traversal_algorithm
TORRENT_TEST(unsorted_traversal_results) TORRENT_TEST(unsorted_traversal_results)
{ {
init_rand_address();
// make sure the handling of an unsorted tail of nodes is correct in the // make sure the handling of an unsorted tail of nodes is correct in the
// traversal algorithm. Initial nodes (that we bootstrap from) remain // traversal algorithm. Initial nodes (that we bootstrap from) remain
// unsorted, since we don't know their node IDs // unsorted, since we don't know their node IDs
@ -3395,7 +3409,7 @@ TORRENT_TEST(unsorted_traversal_results)
TORRENT_TEST(rpc_invalid_error_msg) TORRENT_TEST(rpc_invalid_error_msg)
{ {
// TODO: 3 use dht_test_setup class to simplify the node setup // TODO: 3 use dht_test_setup class to simplify the node setup
dht::dht_settings sett = test_settings(); dht::settings sett = test_settings();
mock_socket s; mock_socket s;
auto ls = dummy_listen_socket4(); auto ls = dummy_listen_socket4();
obs observer; obs observer;
@ -3460,6 +3474,8 @@ TORRENT_TEST(rpc_invalid_error_msg)
// test bucket distribution // test bucket distribution
TORRENT_TEST(node_id_bucket_distribution) TORRENT_TEST(node_id_bucket_distribution)
{ {
init_rand_address();
int nodes_per_bucket[160] = {0}; int nodes_per_bucket[160] = {0};
dht::node_id reference_id = generate_id(rand_v4()); dht::node_id reference_id = generate_id(rand_v4());
int const num_samples = 100000; int const num_samples = 100000;
@ -3519,7 +3535,7 @@ TORRENT_TEST(dht_verify_node_address)
{ {
obs observer; obs observer;
// initial setup taken from dht test above // initial setup taken from dht test above
dht::dht_settings s; dht::settings s;
s.extended_routing_table = false; s.extended_routing_table = false;
node_id id = to_hash("3123456789abcdef01232456789abcdef0123456"); node_id id = to_hash("3123456789abcdef01232456789abcdef0123456");
const int bucket_size = 10; const int bucket_size = 10;
@ -3697,6 +3713,8 @@ TORRENT_TEST(dht_state)
TORRENT_TEST(sample_infohashes) TORRENT_TEST(sample_infohashes)
{ {
init_rand_address();
dht_test_setup t(rand_udp_ep()); dht_test_setup t(rand_udp_ep());
bdecode_node response; bdecode_node response;
@ -3763,6 +3781,59 @@ TORRENT_TEST(sample_infohashes)
TEST_CHECK(g_sent_packets.empty()); TEST_CHECK(g_sent_packets.empty());
} }
namespace {
node_entry fake_node(bool verified, int rtt = 0)
{
node_entry e(rand_udp_ep());
e.verified = verified;
e.rtt = static_cast<std::uint16_t>(rtt);
return e;
}
}
TORRENT_TEST(node_entry_comparison)
{
// being verified or not always trumps RTT in sort order
TEST_CHECK(fake_node(true, 10) < fake_node(false, 5));
TEST_CHECK(fake_node(true, 5) < fake_node(false, 10));
TEST_CHECK(!(fake_node(false, 10) < fake_node(true, 5)));
TEST_CHECK(!(fake_node(false, 5) < fake_node(true, 10)));
// if both are verified, lower RTT is better
TEST_CHECK(fake_node(true, 5) < fake_node(true, 10));
TEST_CHECK(!(fake_node(true, 10) < fake_node(true, 5)));
// if neither are verified, lower RTT is better
TEST_CHECK(fake_node(false, 5) < fake_node(false, 10));
TEST_CHECK(!(fake_node(false, 10) < fake_node(false, 5)));
}
TORRENT_TEST(mostly_verified_nodes)
{
// an empty bucket is OK
TEST_CHECK(mostly_verified_nodes({}));
TEST_CHECK(mostly_verified_nodes({fake_node(true)}));
TEST_CHECK(mostly_verified_nodes({fake_node(true), fake_node(false)}));
TEST_CHECK(mostly_verified_nodes({fake_node(true), fake_node(true), fake_node(false)}));
TEST_CHECK(mostly_verified_nodes({fake_node(true), fake_node(true), fake_node(true), fake_node(false)}));
// a large bucket with only half of the nodes verified, does not count as
// "mostly"
TEST_CHECK(!mostly_verified_nodes({fake_node(true), fake_node(false)
, fake_node(true), fake_node(false)
, fake_node(true), fake_node(false)
, fake_node(true), fake_node(false)
, fake_node(true), fake_node(false)
, fake_node(true), fake_node(false)}));
// 1 of 3 is not "mostly"
TEST_CHECK(!mostly_verified_nodes({fake_node(false), fake_node(true), fake_node(false)}));
TEST_CHECK(!mostly_verified_nodes({fake_node(false)}));
TEST_CHECK(!mostly_verified_nodes({fake_node(false), fake_node(false)}));
TEST_CHECK(!mostly_verified_nodes({fake_node(false), fake_node(false), fake_node(false)}));
}
// TODO: test obfuscated_get_peers // TODO: test obfuscated_get_peers
#else #else

View File

@ -43,7 +43,8 @@ POSSIBILITY OF SUCH DAMAGE.
#include "setup_transfer.hpp" #include "setup_transfer.hpp"
using namespace lt; using namespace lt;
using namespace lt::dht; using dht::dht_storage_interface;
using dht::dht_state;
namespace namespace
{ {