From 3d4ed9f37ff21904f8fa5789a676afb8f928ea35 Mon Sep 17 00:00:00 2001 From: Thomas Yuan Date: Tue, 1 Sep 2015 17:07:38 -0400 Subject: [PATCH 1/5] Add read-only support in dht_settings and outgoing query messages. --- include/libtorrent/kademlia/rpc_manager.hpp | 4 +++- include/libtorrent/session_settings.hpp | 10 ++++++++++ src/kademlia/node.cpp | 6 +++++- src/kademlia/rpc_manager.cpp | 8 +++++++- 4 files changed, 25 insertions(+), 3 deletions(-) diff --git a/include/libtorrent/kademlia/rpc_manager.hpp b/include/libtorrent/kademlia/rpc_manager.hpp index 50120af4c..d98e2ffc2 100644 --- a/include/libtorrent/kademlia/rpc_manager.hpp +++ b/include/libtorrent/kademlia/rpc_manager.hpp @@ -83,7 +83,8 @@ public: rpc_manager(node_id const& our_id , routing_table& table , udp_socket_interface* sock - , dht_logger* log); + , dht_logger* log + , bool read_only = false); ~rpc_manager(); void unreachable(udp::endpoint const& ep); @@ -130,6 +131,7 @@ private: node_id m_our_id; boost::uint32_t m_allocated_observers:31; boost::uint32_t m_destructing:1; + bool m_read_only; }; } } // namespace libtorrent::dht diff --git a/include/libtorrent/session_settings.hpp b/include/libtorrent/session_settings.hpp index b8759ecc1..f6f0522c1 100644 --- a/include/libtorrent/session_settings.hpp +++ b/include/libtorrent/session_settings.hpp @@ -1403,6 +1403,7 @@ namespace libtorrent , ignore_dark_internet(true) , block_timeout(5 * 60) , block_ratelimit(5) + , read_only(false) {} // the maximum number of peers to send in a reply to ``get_peers`` @@ -1488,6 +1489,15 @@ namespace libtorrent // the max number of packets per second a DHT node is allowed to send // without getting banned. int block_ratelimit; + + // when set, the other nodes won't kept this node in their routing + // tables, it's meant for low-power and/or ephemeral devices that + // cannot support the DHT, it is also useful for mobile devices which + // are sensitive to network traffic and battery life. + // this node no longer responds to 'query' messages, and will place a + // 'ro' key (value = 1) in the top-level message dictionary of outgoing + // query messages. + bool read_only; }; diff --git a/src/kademlia/node.cpp b/src/kademlia/node.cpp index aae696f7d..a14a1f574 100644 --- a/src/kademlia/node.cpp +++ b/src/kademlia/node.cpp @@ -109,7 +109,7 @@ node::node(udp_socket_interface* sock : m_settings(settings) , m_id(calculate_node_id(nid, observer)) , m_table(m_id, 8, settings, observer) - , m_rpc(m_id, m_table, sock, observer) + , m_rpc(m_id, m_table, sock, observer, settings.read_only) , m_observer(observer) , m_last_tracker_tick(aux::time_now()) , m_last_self_refresh(min_time()) @@ -280,6 +280,10 @@ void node::incoming(msg const& m) case 'q': { TORRENT_ASSERT(m.message.dict_find_string_value("y") == "q"); + // When a DHT node enters the read-only state, it no longer + // responds to 'query' messages that it receives. + if (m_settings.read_only) break; + entry e; incoming_request(m, e); m_sock->send_packet(e, m.addr, 0); diff --git a/src/kademlia/rpc_manager.cpp b/src/kademlia/rpc_manager.cpp index 2b56f0fb5..633c7d948 100644 --- a/src/kademlia/rpc_manager.cpp +++ b/src/kademlia/rpc_manager.cpp @@ -164,7 +164,8 @@ enum { observer_size = max3< rpc_manager::rpc_manager(node_id const& our_id , routing_table& table, udp_socket_interface* sock - , dht_logger* log) + , dht_logger* log + , bool read_only) : m_pool_allocator(observer_size, 10) , m_sock(sock) , m_log(log) @@ -173,6 +174,7 @@ rpc_manager::rpc_manager(node_id const& our_id , m_our_id(our_id) , m_allocated_observers(0) , m_destructing(false) + , m_read_only(read_only) {} rpc_manager::~rpc_manager() @@ -439,6 +441,10 @@ bool rpc_manager::invoke(entry& e, udp::endpoint target_addr io::write_uint16(tid, out); e["t"] = transaction_id; + // When a DHT node enters the read-only state, in each outgoing query message, + // places a 'ro' key in the top-level message dictionary and sets its value to 1. + if (m_read_only) e["ro"] = 1; + o->set_target(target_addr); o->set_transaction_id(tid); From d6bb387ab96ed1f562ba9981a4c91a050a2e7f8f Mon Sep 17 00:00:00 2001 From: Thomas Yuan Date: Wed, 2 Sep 2015 18:09:49 -0400 Subject: [PATCH 2/5] Use dht_settings directly instead of add a read_only member variable. Since rpc_manager has a reference of dht_settings, needn't pass it as a parameter for incoming(). --- include/libtorrent/kademlia/rpc_manager.hpp | 8 ++++---- include/libtorrent/session_settings.hpp | 2 +- src/kademlia/node.cpp | 6 +++--- src/kademlia/rpc_manager.cpp | 13 ++++++------- 4 files changed, 14 insertions(+), 15 deletions(-) diff --git a/include/libtorrent/kademlia/rpc_manager.hpp b/include/libtorrent/kademlia/rpc_manager.hpp index d98e2ffc2..6cd27c002 100644 --- a/include/libtorrent/kademlia/rpc_manager.hpp +++ b/include/libtorrent/kademlia/rpc_manager.hpp @@ -81,17 +81,17 @@ class TORRENT_EXTRA_EXPORT rpc_manager public: rpc_manager(node_id const& our_id + , dht_settings const& settings , routing_table& table , udp_socket_interface* sock - , dht_logger* log - , bool read_only = false); + , dht_logger* log); ~rpc_manager(); void unreachable(udp::endpoint const& ep); // returns true if the node needs a refresh // if so, id is assigned the node id to refresh - bool incoming(msg const&, node_id* id, libtorrent::dht_settings const& settings); + bool incoming(msg const&, node_id* id); time_duration tick(); bool invoke(entry& e, udp::endpoint target @@ -126,12 +126,12 @@ private: udp_socket_interface* m_sock; dht_logger* m_log; + dht_settings const& m_settings; routing_table& m_table; time_point m_timer; node_id m_our_id; boost::uint32_t m_allocated_observers:31; boost::uint32_t m_destructing:1; - bool m_read_only; }; } } // namespace libtorrent::dht diff --git a/include/libtorrent/session_settings.hpp b/include/libtorrent/session_settings.hpp index f6f0522c1..393ea3867 100644 --- a/include/libtorrent/session_settings.hpp +++ b/include/libtorrent/session_settings.hpp @@ -1490,7 +1490,7 @@ namespace libtorrent // without getting banned. int block_ratelimit; - // when set, the other nodes won't kept this node in their routing + // when set, the other nodes won't keep this node in their routing // tables, it's meant for low-power and/or ephemeral devices that // cannot support the DHT, it is also useful for mobile devices which // are sensitive to network traffic and battery life. diff --git a/src/kademlia/node.cpp b/src/kademlia/node.cpp index a14a1f574..151c883bc 100644 --- a/src/kademlia/node.cpp +++ b/src/kademlia/node.cpp @@ -109,7 +109,7 @@ node::node(udp_socket_interface* sock : m_settings(settings) , m_id(calculate_node_id(nid, observer)) , m_table(m_id, 8, settings, observer) - , m_rpc(m_id, m_table, sock, observer, settings.read_only) + , m_rpc(m_id, m_settings, m_table, sock, observer) , m_observer(observer) , m_last_tracker_tick(aux::time_now()) , m_last_self_refresh(min_time()) @@ -274,7 +274,7 @@ void node::incoming(msg const& m) case 'r': { node_id id; - m_rpc.incoming(m, &id, m_settings); + m_rpc.incoming(m, &id); break; } case 'q': @@ -300,7 +300,7 @@ void node::incoming(msg const& m) } #endif node_id id; - m_rpc.incoming(m, &id, m_settings); + m_rpc.incoming(m, &id); break; } } diff --git a/src/kademlia/rpc_manager.cpp b/src/kademlia/rpc_manager.cpp index 633c7d948..a97ad0836 100644 --- a/src/kademlia/rpc_manager.cpp +++ b/src/kademlia/rpc_manager.cpp @@ -163,18 +163,18 @@ enum { observer_size = max3< }; rpc_manager::rpc_manager(node_id const& our_id + , dht_settings const& settings , routing_table& table, udp_socket_interface* sock - , dht_logger* log - , bool read_only) + , dht_logger* log) : m_pool_allocator(observer_size, 10) , m_sock(sock) , m_log(log) + , m_settings(settings) , m_table(table) , m_timer(aux::time_now()) , m_our_id(our_id) , m_allocated_observers(0) , m_destructing(false) - , m_read_only(read_only) {} rpc_manager::~rpc_manager() @@ -246,8 +246,7 @@ void rpc_manager::unreachable(udp::endpoint const& ep) } } -bool rpc_manager::incoming(msg const& m, node_id* id - , libtorrent::dht_settings const& settings) +bool rpc_manager::incoming(msg const& m, node_id* id) { INVARIANT_CHECK; @@ -337,7 +336,7 @@ bool rpc_manager::incoming(msg const& m, node_id* id } node_id nid = node_id(node_id_ent.string_ptr()); - if (settings.enforce_node_id && !verify_id(nid, m.addr.address())) + if (m_settings.enforce_node_id && !verify_id(nid, m.addr.address())) { o->timeout(); return false; @@ -443,7 +442,7 @@ bool rpc_manager::invoke(entry& e, udp::endpoint target_addr // When a DHT node enters the read-only state, in each outgoing query message, // places a 'ro' key in the top-level message dictionary and sets its value to 1. - if (m_read_only) e["ro"] = 1; + if (m_settings.read_only) e["ro"] = 1; o->set_target(target_addr); o->set_transaction_id(tid); From 94a2c3131f8fcb635fb36841a53f6a65128c7348 Mon Sep 17 00:00:00 2001 From: Thomas Yuan Date: Thu, 3 Sep 2015 18:49:59 -0400 Subject: [PATCH 3/5] Add unit test for DHT read_only nodes. --- test/test_dht.cpp | 160 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 160 insertions(+) diff --git a/test/test_dht.cpp b/test/test_dht.cpp index 1d7d942e2..36db16104 100644 --- a/test/test_dht.cpp +++ b/test/test_dht.cpp @@ -136,6 +136,57 @@ void lazy_from_entry(entry const& e, bdecode_node& l) TEST_CHECK(ret == 0); } +void send_simple_dht_request(node& node, char const* msg, udp::endpoint const& ep + , bdecode_node* reply, entry const& args, + char const* t = "10", bool has_response = true) +{ + reply->clear(); + entry e; + e["q"] = msg; + e["t"] = t; + e["y"] = "q"; + e["a"] = args; + + char msg_buf[1500]; + int size = bencode(msg_buf, e); + +#ifdef TORRENT_USE_VALGRIND + VALGRIND_CHECK_MEM_IS_DEFINED(msg_buf, size); +#endif + + bdecode_node decoded; + error_code ec; + bdecode(msg_buf, msg_buf + size, decoded, ec); + TEST_CHECK(!ec); + + dht::msg m(decoded, ep); + node.incoming(m); + + // If the request is supposed to get a response, by now the node should have + // invoked the send function and put the response in g_sent_packets + std::list >::iterator i = find_packet(ep); + if (has_response) + { + if (i == g_sent_packets.end()) + { + TEST_ERROR("not response from DHT node"); + return; + } + lazy_from_entry(i->second, *reply); + g_sent_packets.erase(i); + + return; + } + + // this request suppose won't be responsed. + if (i != g_sent_packets.end()) + { + TEST_ERROR("shouldn't have response from DHT node"); + return; + } +} + + void send_dht_request(node& node, char const* msg, udp::endpoint const& ep , bdecode_node* reply, char const* t = "10", char const* info_hash = 0 , char const* name = 0, std::string const token = std::string(), int port = 0 @@ -451,6 +502,16 @@ dht_settings test_settings() return sett; } +entry test_args(sha1_hash const* nid = NULL) +{ + entry a; + + if (nid == NULL) a["id"] = generate_next().to_string(); + else a["id"] = nid->to_string(); + + return a; +} + // TODO: test obfuscated_get_peers // TODO: 2 split this test up into smaller test cases TORRENT_TEST(dht) @@ -1334,6 +1395,7 @@ TORRENT_TEST(dht) {"target", bdecode_node::string_t, 20, key_desc_t::last_child}, }; + // bootstrap g_sent_packets.clear(); @@ -2059,5 +2121,103 @@ TORRENT_TEST(routing_table_extended) #endif } +TORRENT_TEST(read_only_node) +{ + dht_settings sett = test_settings(); + sett.read_only = true; + mock_socket s; + obs observer; + counters cnt; + + dht::node node(&s, sett, node_id(0), &observer, cnt); + udp::endpoint source(address::from_string("10.0.0.1"), 20); + bdecode_node response; + entry args = test_args(); + + // for incoming requests, read_only node won't response. + send_simple_dht_request(node, "ping", source, &response, args, "10", false); + TEST_EQUAL(response.type(), bdecode_node::none_t); + + args["target"] = "01010101010101010101"; + send_simple_dht_request(node, "get", source, &response, args, "10", false); + TEST_EQUAL(response.type(), bdecode_node::none_t); + + // also, the sender shouldn't be added to routing table. + boost::tuple nums = node.size(); + TEST_EQUAL(nums.get<0>(), 0); + TEST_EQUAL(nums.get<1>(), 0); + TEST_EQUAL(nums.get<2>(), 0); + + // for outgoing requests, read_only node will add 'ro' key (value == 1) + // in top-level of request. + bdecode_node parsed[7]; + char error_string[200]; + udp::endpoint initial_node(address_v4::from_string("4.4.4.4"), 1234); + node.m_table.add_node(initial_node); + bdecode_node request; + sha1_hash target = generate_next(); + + node.get_item(target, get_item_cb); + TEST_EQUAL(g_sent_packets.size(), 1); + TEST_EQUAL(g_sent_packets.front().first, initial_node); + + dht::key_desc_t get_item_desc[] = { + {"y", bdecode_node::string_t, 1, 0}, + {"t", bdecode_node::string_t, 2, 0}, + {"q", bdecode_node::string_t, 3, 0}, + {"ro", bdecode_node::int_t, 4, key_desc_t::optional}, + {"a", bdecode_node::dict_t, 0, key_desc_t::parse_children}, + {"id", bdecode_node::string_t, 20, 0}, + {"target", bdecode_node::string_t, 20, key_desc_t::last_child}, + }; + + lazy_from_entry(g_sent_packets.front().second, request); + bool ret = verify_message(request, get_item_desc, parsed, 7, error_string + , sizeof(error_string)); + + TEST_CHECK(ret); + TEST_EQUAL(parsed[3].int_value(), 1); + + // should have one node now, whichi is 4.4.4.4:1234 + nums = node.size(); + TEST_EQUAL(nums.get<0>(), 1); + TEST_EQUAL(nums.get<1>(), 0); + TEST_EQUAL(nums.get<2>(), 0); + + // now, disable read_only, try again. + g_sent_packets.clear(); + sett.read_only = false; + + send_simple_dht_request(node, "get", source, &response, args, "10", true); + // sender should be added to routing table + nums = node.size(); + TEST_EQUAL(nums.get<0>(), 2); + TEST_EQUAL(nums.get<1>(), 0); + TEST_EQUAL(nums.get<2>(), 0); + + // now, request shouldn't have 'ro' key anymore + g_sent_packets.clear(); + target = generate_next(); + node.get_item(target, get_item_cb); + + // since we have 2 nodes, we should have two packets. + TEST_EQUAL(g_sent_packets.size(), 2); + + // both of them shouldn't a 'ro' key. + lazy_from_entry(g_sent_packets.front().second, request); + ret = verify_message(request, get_item_desc, parsed, 7, error_string + , sizeof(error_string)); + + TEST_CHECK(ret); + TEST_CHECK(!parsed[3]); + + lazy_from_entry(g_sent_packets.back().second, request); + ret = verify_message(request, get_item_desc, parsed, 7, error_string + , sizeof(error_string)); + + TEST_CHECK(ret); + TEST_CHECK(!parsed[3]); +} + #endif From 4d6de4bcd8c716a1367b2b2e630a386f5243c6a3 Mon Sep 17 00:00:00 2001 From: Thomas Yuan Date: Fri, 4 Sep 2015 10:11:27 -0400 Subject: [PATCH 4/5] save/load all dht_settings. --- src/session_impl.cpp | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/src/session_impl.cpp b/src/session_impl.cpp index c3d6448e5..3459d2c7a 100644 --- a/src/session_impl.cpp +++ b/src/session_impl.cpp @@ -704,7 +704,15 @@ namespace aux { dht_sett["max_dht_items"] = m_dht_settings.max_dht_items; dht_sett["max_torrent_search_reply"] = m_dht_settings.max_torrent_search_reply; dht_sett["restrict_routing_ips"] = m_dht_settings.restrict_routing_ips; + dht_sett["restrict_search_ips"] = m_dht_settings.restrict_search_ips; dht_sett["extended_routing_table"] = m_dht_settings.extended_routing_table; + dht_sett["aggressive_lookups"] = m_dht_settings.aggressive_lookups; + dht_sett["privacy_lookups"] = m_dht_settings.privacy_lookups; + dht_sett["enforce_node_id"] = m_dht_settings.enforce_node_id; + dht_sett["ignore_dark_internet"] = m_dht_settings.ignore_dark_internet; + dht_sett["block_timeout"] = m_dht_settings.block_timeout; + dht_sett["block_ratelimit"] = m_dht_settings.block_ratelimit; + dht_sett["read_only"] = m_dht_settings.read_only; } if (m_dht && (flags & session::save_dht_state)) @@ -769,8 +777,24 @@ namespace aux { if (val) m_dht_settings.max_torrent_search_reply = val.int_value(); val = settings.dict_find_int("restrict_routing_ips"); if (val) m_dht_settings.restrict_routing_ips = val.int_value(); + val = settings.dict_find_int("restrict_search_ips"); + if (val) m_dht_settings.restrict_search_ips = val.int_value(); val = settings.dict_find_int("extended_routing_table"); if (val) m_dht_settings.extended_routing_table = val.int_value(); + val = settings.dict_find_int("aggressive_lookups"); + if (val) m_dht_settings.aggressive_lookups = val.int_value(); + val = settings.dict_find_int("privacy_lookups"); + if (val) m_dht_settings.privacy_lookups = val.int_value(); + val = settings.dict_find_int("enforce_node_id"); + if (val) m_dht_settings.enforce_node_id = val.int_value(); + val = settings.dict_find_int("ignore_dark_internet"); + if (val) m_dht_settings.ignore_dark_internet = val.int_value(); + val = settings.dict_find_int("block_timeout"); + if (val) m_dht_settings.block_timeout = val.int_value(); + val = settings.dict_find_int("block_ratelimit"); + if (val) m_dht_settings.block_ratelimit = val.int_value(); + val = settings.dict_find_int("read_only"); + if (val) m_dht_settings.read_only = val.int_value(); } #endif From 60cedf881f1a0ead861ea824a120e028a5dee848 Mon Sep 17 00:00:00 2001 From: Thomas Yuan Date: Fri, 4 Sep 2015 14:28:38 -0400 Subject: [PATCH 5/5] Add block_timeout, block_ratelimit and read_only of dht_settings to python binding. --- bindings/python/src/session_settings.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/bindings/python/src/session_settings.cpp b/bindings/python/src/session_settings.cpp index a8dc7f526..a5c2a662b 100644 --- a/bindings/python/src/session_settings.cpp +++ b/bindings/python/src/session_settings.cpp @@ -290,6 +290,9 @@ void bind_session_settings() .def_readwrite("privacy_lookups", &dht_settings::privacy_lookups) .def_readwrite("enforce_node_id", &dht_settings::enforce_node_id) .def_readwrite("ignore_dark_internet", &dht_settings::ignore_dark_internet) + .def_readwrite("block_timeout", &dht_settings::block_timeout) + .def_readwrite("block_ratelimit", &dht_settings::block_ratelimit) + .def_readwrite("read_only", &dht_settings::read_only) ; #endif