From 15ec839d33819167deb47a16b93180e7cc9efc5e Mon Sep 17 00:00:00 2001 From: arvidn Date: Sat, 19 Dec 2015 02:09:06 -0500 Subject: [PATCH 1/4] extend the unit test of dht_storage --- include/libtorrent/session_settings.hpp | 36 +++++---- src/kademlia/dht_storage.cpp | 34 +++++--- test/setup_transfer.cpp | 8 ++ test/setup_transfer.hpp | 2 + test/test_dht_storage.cpp | 103 +++++++++++++++++++++--- 5 files changed, 143 insertions(+), 40 deletions(-) diff --git a/include/libtorrent/session_settings.hpp b/include/libtorrent/session_settings.hpp index 2391550c4..2acbb5dd9 100644 --- a/include/libtorrent/session_settings.hpp +++ b/include/libtorrent/session_settings.hpp @@ -114,13 +114,13 @@ namespace libtorrent // should take for the other end to send all the pieces. i.e. the actual // number of requests depends on the download rate and this number. int request_queue_time; - + // the number of outstanding block requests a peer is allowed to queue up // in the client. If a peer sends more requests than this (before the // first one has been sent) the last request will be dropped. the higher // this is, the faster upload speeds the client can get to a single peer. int max_allowed_in_request_queue; - + // the maximum number of outstanding requests to send to a peer. This // limit takes precedence over request_queue_time. i.e. no matter the // download speed, the number of outstanding requests will never exceed @@ -133,18 +133,18 @@ namespace libtorrent // doing localized accesses and also to make it easier to identify bad // peers if a piece fails the hash check. int whole_pieces_threshold; - + // the number of seconds to wait for any activity on the peer wire before // closing the connectiong due to time out. This defaults to 120 seconds, // since that's what's specified in the protocol specification. After // half the time out, a keep alive message is sent. int peer_timeout; - + // same as peer_timeout, but only applies to url-seeds. this is usually // set lower, because web servers are expected to be more reliable. This // value defaults to 20 seconds. int urlseed_timeout; - + // controls the pipelining with the web server. When using persistent // connections to HTTP 1.1 servers, the client is allowed to send more // requests before the first response is received. This number controls @@ -154,7 +154,7 @@ namespace libtorrent // time to wait until a new retry takes place int urlseed_wait_retry; - + // sets the upper limit on the total number of files this session will // keep open. The reason why files are left open at all is that some anti // virus software hooks on every file close, and scans the file for @@ -165,7 +165,7 @@ namespace libtorrent // this limit and set the number of connections and the number of files // limits so their sum is slightly below it. int file_pool_size; - + // determines if connections from the same IP address as existing // connections should be rejected or not. Multiple connections from the // same IP address is not allowed by default, to prevent abusive behavior @@ -179,7 +179,7 @@ namespace libtorrent // retrieved from a peer source (other than DHT) the failcount is // decremented by one, allowing another try. int max_failcount; - + // the number of seconds to wait to reconnect to a peer. this time is // multiplied with the failcount. int min_reconnect_time; @@ -401,11 +401,11 @@ namespace libtorrent // download to trade with each other. anti_leech }; - + // controls the seeding unchoke behavior. For options, see // seed_choking_algorithm_t. int seed_choking_algorithm; - + // specifies if parole mode should be used. Parole mode means that peers // that participate in pieces that fail the hash check are put in a mode // where they are only allowed to download whole pieces. If the whole @@ -580,7 +580,7 @@ namespace libtorrent // slots to, the default is false which gives preference to downloading // torrents bool auto_manage_prefer_seeds; - + // if true, torrents without any payload transfers are not subject to the // ``active_seeds`` and ``active_downloads`` limits. This is intended to // make it more likely to utilize all available bandwidth, and avoid @@ -590,7 +590,7 @@ namespace libtorrent // the number of seconds in between recalculating which torrents to // activate and which ones to queue int auto_manage_interval; - + // when a seeding torrent reaches either the share ratio (bytes up / // bytes down) or the seed time ratio (seconds as seed / seconds as // downloader) or the seed time limit (seconds as seed) it is considered @@ -1116,7 +1116,7 @@ namespace libtorrent // the number of SYN packets that are sent (and timed out) before // giving up and closing the socket. int utp_syn_resends; - + // the number of resent packets sent on a closed socket before giving up int utp_fin_resends; @@ -1335,7 +1335,7 @@ namespace libtorrent // enables banning web seeds. By default, web seeds that send corrupt // data are banned. bool ban_web_seeds; - + // specifies the max number of bytes to receive into RAM buffers when // downloading stuff over HTTP. Specifically when specifying a URL to a // .torrent file when adding a torrent or when announcing to an HTTP @@ -1393,6 +1393,7 @@ namespace libtorrent , max_fail_count(20) , max_torrents(2000) , max_dht_items(700) + , max_peers(5000) , max_torrent_search_reply(20) , restrict_routing_ips(true) , restrict_search_ips(true) @@ -1436,6 +1437,9 @@ namespace libtorrent // max number of items the DHT will store int max_dht_items; + // the max number of peers to store per torrent (for the DHT) + int max_peers; + // the max number of torrents to return in a torrent search query to the // DHT int max_torrent_search_reply; @@ -1533,7 +1537,7 @@ namespace libtorrent // and if an outgoing encrypted connection fails, a non- encrypted // connection will be tried. enabled, - + // only non-encrypted connections are allowed. disabled }; @@ -1543,7 +1547,7 @@ namespace libtorrent { // use only plaintext encryption plaintext = 1, - // use only rc4 encryption + // use only rc4 encryption rc4 = 2, // allow both both = 3 diff --git a/src/kademlia/dht_storage.cpp b/src/kademlia/dht_storage.cpp index 90e16c764..b26fce846 100644 --- a/src/kademlia/dht_storage.cpp +++ b/src/kademlia/dht_storage.cpp @@ -279,7 +279,7 @@ namespace int num_peers = m_map.begin()->second.peers.size(); table_t::iterator candidate = m_map.begin(); for (table_t::iterator i = m_map.begin() - , end(m_map.end()); i != end; ++i) + , end(m_map.end()); i != end; ++i) { if (int(i->second.peers.size()) > num_peers) continue; if (i->first == info_hash) continue; @@ -303,7 +303,7 @@ namespace if (!name.empty() && v->name.empty()) { std::string tname = name; - if (tname.size() > 50) tname.resize(50); + if (tname.size() > 100) tname.resize(100); v->name = tname; } @@ -317,6 +317,16 @@ namespace v->peers.erase(i++); m_counters.peers -= 1; } + else if (v->peers.size() >= m_settings.max_peers) + { + // when we're at capacity, there's a 50/50 chance of dropping the + // announcing peer or an existing peer + if (random() & 1) return; + i = v->peers.lower_bound(peer); + if (i == v->peers.end()) --i; + i = v->peers.erase(i); + m_counters.peers -= 1; + } v->peers.insert(i, peer); m_counters.peers += 1; } @@ -475,19 +485,17 @@ namespace for (table_t::iterator i = m_map.begin(), end(m_map.end()); i != end;) { torrent_entry& t = i->second; - node_id const& key = i->first; - ++i; purge_peers(t.peers); - if (!t.peers.empty()) continue; + if (!t.peers.empty()) + { + ++i; + continue; + } // if there are no more peers, remove the entry altogether - table_t::iterator it = m_map.find(key); - if (it != m_map.end()) - { - m_map.erase(it); - m_counters.torrents -= 1;// peers is decreased by purge_peers - } + i = m_map.erase(i); + m_counters.torrents -= 1;// peers is decreased by purge_peers } if (0 == m_settings.item_lifetime) return; @@ -505,7 +513,7 @@ namespace continue; } free(i->second.value); - m_immutable_table.erase(i++); + i = m_immutable_table.erase(i); m_counters.immutable_data -= 1; } @@ -519,7 +527,7 @@ namespace } free(i->second.value); free(i->second.salt); - m_mutable_table.erase(i++); + i = m_mutable_table.erase(i); m_counters.mutable_data -= 1; } } diff --git a/test/setup_transfer.cpp b/test/setup_transfer.cpp index 96367e87d..6f8a4af8c 100644 --- a/test/setup_transfer.cpp +++ b/test/setup_transfer.cpp @@ -94,6 +94,14 @@ address rand_v4() return address_v4(g_addr); } +sha1_hash rand_hash() +{ + sha1_hash ret; + for (int i = 0; i < 20; ++i) + ret[i] = lt::random(); + return ret; +} + #if TORRENT_USE_IPV6 address rand_v6() { diff --git a/test/setup_transfer.hpp b/test/setup_transfer.hpp index 4c223f73c..8a2395541 100644 --- a/test/setup_transfer.hpp +++ b/test/setup_transfer.hpp @@ -59,6 +59,8 @@ EXPORT libtorrent::address rand_v6(); EXPORT libtorrent::tcp::endpoint rand_tcp_ep(); EXPORT libtorrent::udp::endpoint rand_udp_ep(); +EXPORT libtorrent::sha1_hash rand_hash(); + EXPORT std::map get_counters(libtorrent::session& s); EXPORT libtorrent::alert const* wait_for_alert( diff --git a/test/test_dht_storage.cpp b/test/test_dht_storage.cpp index 64b7c02a0..4c4011f89 100644 --- a/test/test_dht_storage.cpp +++ b/test/test_dht_storage.cpp @@ -47,6 +47,7 @@ POSSIBILITY OF SUCH DAMAGE. #include "libtorrent/kademlia/routing_table.hpp" #include "libtorrent/kademlia/item.hpp" #include "libtorrent/kademlia/dht_observer.hpp" +#include "libtorrent/random.hpp" #include "libtorrent/ed25519.hpp" #include @@ -83,18 +84,17 @@ namespace } } -TORRENT_TEST(dht_storage) +const sha1_hash n1 = to_hash("5fbfbff10c5d6a4ec8a88e4c6ab4c28b95eee401"); +const sha1_hash n2 = to_hash("5fbfbff10c5d6a4ec8a88e4c6ab4c28b95eee402"); +const sha1_hash n3 = to_hash("5fbfbff10c5d6a4ec8a88e4c6ab4c28b95eee403"); +const sha1_hash n4 = to_hash("5fbfbff10c5d6a4ec8a88e4c6ab4c28b95eee404"); + +TORRENT_TEST(announce_peer) { dht_settings sett = test_settings(); boost::scoped_ptr s(dht_default_storage_constructor(node_id(0), sett)); - TEST_CHECK(s.get() != NULL); - sha1_hash n1 = to_hash("5fbfbff10c5d6a4ec8a88e4c6ab4c28b95eee401"); - sha1_hash n2 = to_hash("5fbfbff10c5d6a4ec8a88e4c6ab4c28b95eee402"); - sha1_hash n3 = to_hash("5fbfbff10c5d6a4ec8a88e4c6ab4c28b95eee403"); - sha1_hash n4 = to_hash("5fbfbff10c5d6a4ec8a88e4c6ab4c28b95eee404"); - entry peers; s->get_peers(n1, false, false, peers); @@ -116,9 +116,16 @@ TORRENT_TEST(dht_storage) s->announce_peer(n3, p4, "torrent_name2", false); bool r = s->get_peers(n1, false, false, peers); TEST_CHECK(!r); +} + +TORRENT_TEST(put_immutable_item) +{ + dht_settings sett = test_settings(); + boost::scoped_ptr s(dht_default_storage_constructor(node_id(0), sett)); + TEST_CHECK(s.get() != NULL); entry item; - r = s->get_immutable_item(n4, item); + bool r = s->get_immutable_item(n4, item); TEST_CHECK(!r); s->put_immutable_item(n4, "123", 3, address::from_string("124.31.75.21")); @@ -141,7 +148,7 @@ TORRENT_TEST(dht_storage) TEST_CHECK(r); } -TORRENT_TEST(dht_storage_counters) +TORRENT_TEST(counters) { dht_settings sett = test_settings(); boost::scoped_ptr s(dht_default_storage_constructor(node_id(0), sett)); @@ -187,13 +194,14 @@ TORRENT_TEST(dht_storage_counters) TEST_EQUAL(s->counters().mutable_data, 1); } -TORRENT_TEST(dht_storage_set_custom) +TORRENT_TEST(set_custom) { g_storage_constructor_invoked = false; settings_pack p; p.set_bool(settings_pack::enable_dht, false); lt::session ses(p); + TEST_EQUAL(g_storage_constructor_invoked, false); bool r = ses.is_dht_running(); TEST_CHECK(!r); @@ -206,7 +214,7 @@ TORRENT_TEST(dht_storage_set_custom) TEST_EQUAL(g_storage_constructor_invoked, true); } -TORRENT_TEST(dht_storage_default_set_custom) +TORRENT_TEST(default_set_custom) { g_storage_constructor_invoked = false; settings_pack p; @@ -233,4 +241,77 @@ TORRENT_TEST(dht_storage_default_set_custom) TEST_EQUAL(g_storage_constructor_invoked, true); } +TORRENT_TEST(peer_limit) +{ + dht_settings sett = test_settings(); + sett.max_peers = 1337; + boost::scoped_ptr s(dht_default_storage_constructor(node_id(0), sett)); + TEST_CHECK(s.get() != NULL); + + for (int i = 0; i < 10000; ++i) + { + s->announce_peer(n1, tcp::endpoint(rand_v4(), lt::random()) + , "torrent_name", false); + dht_storage_counters cnt = s->counters(); + TEST_CHECK(cnt.peers <= 1337); + } + dht_storage_counters cnt = s->counters(); + TEST_EQUAL(cnt.peers, 1337); +} + +TORRENT_TEST(torrent_limit) +{ + dht_settings sett = test_settings(); + sett.max_torrents = 1337; + boost::scoped_ptr s(dht_default_storage_constructor(node_id(0), sett)); + TEST_CHECK(s.get() != NULL); + + for (int i = 0; i < 10000; ++i) + { + s->announce_peer(rand_hash(), tcp::endpoint(rand_v4(), lt::random()) + , "", false); + dht_storage_counters cnt = s->counters(); + TEST_CHECK(cnt.torrents <= 1337); + } + dht_storage_counters cnt = s->counters(); + TEST_EQUAL(cnt.torrents, 1337); +} + +TORRENT_TEST(immutable_item_limit) +{ + dht_settings sett = test_settings(); + sett.max_dht_items = 1337; + boost::scoped_ptr s(dht_default_storage_constructor(node_id(0), sett)); + TEST_CHECK(s.get() != NULL); + + for (int i = 0; i < 10000; ++i) + { + s->put_immutable_item(rand_hash(), "123", 3, rand_v4()); + dht_storage_counters cnt = s->counters(); + TEST_CHECK(cnt.immutable_data <= 1337); + } + dht_storage_counters cnt = s->counters(); + TEST_EQUAL(cnt.immutable_data, 1337); +} + +TORRENT_TEST(mutable_item_limit) +{ + dht_settings sett = test_settings(); + sett.max_dht_items = 1337; + boost::scoped_ptr s(dht_default_storage_constructor(node_id(0), sett)); + TEST_CHECK(s.get() != NULL); + + char public_key[item_pk_len]; + char signature[item_sig_len]; + for (int i = 0; i < 10000; ++i) + { + s->put_mutable_item(rand_hash(), "123", 3, signature, 1, public_key, "salt", 4, rand_v4()); + dht_storage_counters cnt = s->counters(); + TEST_CHECK(cnt.mutable_data <= 1337); + } + dht_storage_counters cnt = s->counters(); + TEST_EQUAL(cnt.mutable_data, 1337); +} + + #endif From 28749466a20c8e81e2f9228309f6479b7beba839 Mon Sep 17 00:00:00 2001 From: arvidn Date: Sat, 19 Dec 2015 02:40:10 -0500 Subject: [PATCH 2/4] fix build --- src/kademlia/dht_storage.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/kademlia/dht_storage.cpp b/src/kademlia/dht_storage.cpp index b26fce846..7665d07d5 100644 --- a/src/kademlia/dht_storage.cpp +++ b/src/kademlia/dht_storage.cpp @@ -324,7 +324,7 @@ namespace if (random() & 1) return; i = v->peers.lower_bound(peer); if (i == v->peers.end()) --i; - i = v->peers.erase(i); + v->peers.erase(i++); m_counters.peers -= 1; } v->peers.insert(i, peer); @@ -494,7 +494,7 @@ namespace } // if there are no more peers, remove the entry altogether - i = m_map.erase(i); + m_map.erase(i++); m_counters.torrents -= 1;// peers is decreased by purge_peers } @@ -513,7 +513,7 @@ namespace continue; } free(i->second.value); - i = m_immutable_table.erase(i); + m_immutable_table.erase(i++); m_counters.immutable_data -= 1; } @@ -527,7 +527,7 @@ namespace } free(i->second.value); free(i->second.salt); - i = m_mutable_table.erase(i); + m_mutable_table.erase(i++); m_counters.mutable_data -= 1; } } From d7b435ec256d7ac24efeed7c4c50d2ca9f570cd3 Mon Sep 17 00:00:00 2001 From: arvidn Date: Sat, 19 Dec 2015 14:59:37 -0500 Subject: [PATCH 3/4] speed up test --- test/test_dht_storage.cpp | 34 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/test/test_dht_storage.cpp b/test/test_dht_storage.cpp index 4c4011f89..dba79c27b 100644 --- a/test/test_dht_storage.cpp +++ b/test/test_dht_storage.cpp @@ -244,74 +244,74 @@ TORRENT_TEST(default_set_custom) TORRENT_TEST(peer_limit) { dht_settings sett = test_settings(); - sett.max_peers = 1337; + sett.max_peers = 42; boost::scoped_ptr s(dht_default_storage_constructor(node_id(0), sett)); TEST_CHECK(s.get() != NULL); - for (int i = 0; i < 10000; ++i) + for (int i = 0; i < 200; ++i) { s->announce_peer(n1, tcp::endpoint(rand_v4(), lt::random()) , "torrent_name", false); dht_storage_counters cnt = s->counters(); - TEST_CHECK(cnt.peers <= 1337); + TEST_CHECK(cnt.peers <= 42); } dht_storage_counters cnt = s->counters(); - TEST_EQUAL(cnt.peers, 1337); + TEST_EQUAL(cnt.peers, 42); } TORRENT_TEST(torrent_limit) { dht_settings sett = test_settings(); - sett.max_torrents = 1337; + sett.max_torrents = 42; boost::scoped_ptr s(dht_default_storage_constructor(node_id(0), sett)); TEST_CHECK(s.get() != NULL); - for (int i = 0; i < 10000; ++i) + for (int i = 0; i < 200; ++i) { s->announce_peer(rand_hash(), tcp::endpoint(rand_v4(), lt::random()) , "", false); dht_storage_counters cnt = s->counters(); - TEST_CHECK(cnt.torrents <= 1337); + TEST_CHECK(cnt.torrents <= 42); } dht_storage_counters cnt = s->counters(); - TEST_EQUAL(cnt.torrents, 1337); + TEST_EQUAL(cnt.torrents, 42); } TORRENT_TEST(immutable_item_limit) { dht_settings sett = test_settings(); - sett.max_dht_items = 1337; + sett.max_dht_items = 42; boost::scoped_ptr s(dht_default_storage_constructor(node_id(0), sett)); TEST_CHECK(s.get() != NULL); - for (int i = 0; i < 10000; ++i) + for (int i = 0; i < 200; ++i) { s->put_immutable_item(rand_hash(), "123", 3, rand_v4()); dht_storage_counters cnt = s->counters(); - TEST_CHECK(cnt.immutable_data <= 1337); + TEST_CHECK(cnt.immutable_data <= 42); } dht_storage_counters cnt = s->counters(); - TEST_EQUAL(cnt.immutable_data, 1337); + TEST_EQUAL(cnt.immutable_data, 42); } TORRENT_TEST(mutable_item_limit) { dht_settings sett = test_settings(); - sett.max_dht_items = 1337; + sett.max_dht_items = 42; boost::scoped_ptr s(dht_default_storage_constructor(node_id(0), sett)); TEST_CHECK(s.get() != NULL); char public_key[item_pk_len]; char signature[item_sig_len]; - for (int i = 0; i < 10000; ++i) + for (int i = 0; i < 200; ++i) { s->put_mutable_item(rand_hash(), "123", 3, signature, 1, public_key, "salt", 4, rand_v4()); dht_storage_counters cnt = s->counters(); - TEST_CHECK(cnt.mutable_data <= 1337); + TEST_CHECK(cnt.mutable_data <= 42); } dht_storage_counters cnt = s->counters(); - TEST_EQUAL(cnt.mutable_data, 1337); + TEST_EQUAL(cnt.mutable_data, 42); } - #endif + From 626baade75005cedd1198765c2333f4205cc9d2e Mon Sep 17 00:00:00 2001 From: arvidn Date: Sat, 19 Dec 2015 22:06:25 -0500 Subject: [PATCH 4/4] fix dht_storage bug --- src/kademlia/dht_storage.cpp | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/kademlia/dht_storage.cpp b/src/kademlia/dht_storage.cpp index 7665d07d5..37fbad3a1 100644 --- a/src/kademlia/dht_storage.cpp +++ b/src/kademlia/dht_storage.cpp @@ -423,10 +423,13 @@ namespace { // delete the least important one (i.e. the one // the fewest peers are announcing) + // TODO: c++11 use a lambda here instead dht_mutable_table_t::iterator j = std::min_element(m_mutable_table.begin() , m_mutable_table.end() , boost::bind(&dht_immutable_item::num_announcers - , boost::bind(&dht_mutable_table_t::value_type::second, _1))); + , boost::bind(&dht_mutable_table_t::value_type::second, _1)) + < boost::bind(&dht_immutable_item::num_announcers + , boost::bind(&dht_mutable_table_t::value_type::second, _2))); TORRENT_ASSERT(j != m_mutable_table.end()); free(j->second.value); free(j->second.salt);