merged changes from RC_1_0

include/libtorrent/aux_/session_impl.hpp

@@ -370,7 +370,7 @@ namespace libtorrent
 			void resume();
 
 			void set_ip_filter(ip_filter const& f);
-			ip_filter const& get_ip_filter() const;
+			ip_filter& get_ip_filter();
 			
 			void set_port_filter(port_filter const& f);
 			port_filter const& get_port_filter() const;

include/libtorrent/aux_/session_interface.hpp

@@ -142,7 +142,7 @@ namespace libtorrent { namespace aux
 		virtual void remove_torrent_impl(boost::shared_ptr<torrent> tptr, int options) = 0;
 
 		// ip and port filter
-		virtual ip_filter const& get_ip_filter() const = 0;
+		virtual ip_filter& get_ip_filter() = 0;
 		virtual port_filter const& get_port_filter() const = 0;
 
 		virtual boost::int64_t session_time() const = 0;

include/libtorrent/kademlia/node_id.hpp

@@ -63,6 +63,7 @@ int TORRENT_EXTRA_EXPORT distance_exp(node_id const& n1, node_id const& n2);
 
 node_id TORRENT_EXTRA_EXPORT generate_id(address const& external_ip);
 node_id TORRENT_EXTRA_EXPORT generate_random_id();
+bool TORRENT_EXTRA_EXPORT verify_random_id(node_id const& nid);
 node_id TORRENT_EXTRA_EXPORT generate_id_impl(address const& ip_, boost::uint32_t r);
 
 bool TORRENT_EXTRA_EXPORT verify_id(node_id const& nid, address const& source_ip);

include/libtorrent/session.hpp

@@ -281,6 +281,8 @@ namespace libtorrent
 		// it to finish. The timeout can be set with apply_settings().
 		~session();
 
+		// TODO: 2 the ip filter should probably be saved here too
+
 		// flags that determines which aspects of the session should be
 		// saved when calling save_state().
 		enum save_state_flags_t

src/kademlia/get_item.cpp

@@ -216,6 +216,8 @@ void get_item::put(std::vector<std::pair<node_entry, std::string> > const& v)
 
 		void* ptr = m_node.m_rpc.allocate_observer();
 		if (ptr == 0) return;
+
+		// TODO: 3 we don't support CAS errors here! we need a custom observer
 		observer_ptr o(new (ptr) announce_observer(algo, i->first.ep(), i->first.id));
 #if TORRENT_USE_ASSERTS
 		o->m_in_constructor = false;

src/kademlia/get_peers.cpp

@@ -229,7 +229,7 @@ bool obfuscated_get_peers::invoke(observer_ptr o)
 
 	entry e;
 	e["y"] = "q";
-	e["q"] = "find_node";
+	e["q"] = "get_peers";
 	entry& a = e["a"];
 
 	// This logic will obfuscate the target info-hash
@@ -238,11 +238,11 @@ bool obfuscated_get_peers::invoke(observer_ptr o)
 	// bits in the info-hash for the node we're querying to
 	// give a good answer, but not more.
 
-	// now, obfuscate the bits past shared_prefix + 5
+	// now, obfuscate the bits past shared_prefix + 3
-	node_id obfuscated_target = generate_random_id();
+	node_id mask = generate_prefix_mask(shared_prefix + 3);
-	obfuscated_target >>= shared_prefix + 3;
+	node_id obfuscated_target = generate_random_id() & ~mask;
-	obfuscated_target^= m_target;
+	obfuscated_target |= m_target & mask;
-	a["target"] = obfuscated_target.to_string();
+	a["info_hash"] = obfuscated_target.to_string();
 
 	m_node.post_alert(new dht_outgoing_get_peers_alert(m_target
 		, obfuscated_target, o->target_ep()));

src/kademlia/node.cpp

@@ -477,14 +477,9 @@ void node_impl::send_single_refresh(udp::endpoint const& ep, int bucket
 	// generate a random node_id within the given bucket
 	// TODO: 2 it would be nice to have a bias towards node-id prefixes that
 	// are missing in the bucket
-	node_id target = generate_random_id();
 	node_id mask = generate_prefix_mask(bucket + 1);
-
+	node_id target = generate_random_id() & ~mask;
-	// target = (target & ~mask) | (root & mask)
+	target |= m_id & mask;
-	node_id root = m_id;
-	root &= mask;
-	target &= ~mask;
-	target |= root;
 
 	// create a dummy traversal_algorithm		
 	// this is unfortunately necessary for the observer

src/kademlia/node_id.cpp

@@ -41,6 +41,7 @@ POSSIBILITY OF SUCH DAMAGE.
 #include "libtorrent/broadcast_socket.hpp" // for is_local et.al
 #include "libtorrent/socket_io.hpp" // for hash_address
 #include "libtorrent/random.hpp" // for random
+#include "libtorrent/hasher.hpp" // for hasher
 
 namespace libtorrent { namespace dht
 {
@@ -148,11 +149,34 @@ node_id generate_id_impl(address const& ip_, boost::uint32_t r)
 	return id;
 }
 
+static boost::uint32_t secret = 0;
+
 node_id generate_random_id()
 {
 	char r[20];
 	for (int i = 0; i < 20; ++i) r[i] = random() & 0xff;
-	return hasher(r, 20).final();
+	node_id ret = hasher(r, 20).final();
+
+	if (secret == 0) secret = (random() % 0xfffffffe) + 1;
+
+	// generate the last 4 bytes as a "signature" of the previous 4 bytes. This
+	// lets us verify whether a hash came from this function or not in the future.
+	hasher h((char*)&secret, 4);
+	h.update((char*)&ret[20-8], 4);
+	sha1_hash secret_hash = h.final();
+	memcpy(&ret[20-4], &secret_hash[0], 4);
+
+	return ret;
+}
+
+bool verify_random_id(node_id const& nid)
+{
+	if (secret == 0) return false;
+
+	hasher h((char*)&secret, 4);
+	h.update((char const*)&nid[20-8], 4);
+	sha1_hash secret_hash = h.final();
+	return memcmp(&nid[20-4], &secret_hash[0], 4) == 0;
 }
 
 // verifies whether a node-id matches the IP it's used from

src/kademlia/rpc_manager.cpp

@@ -328,7 +328,7 @@ bool rpc_manager::incoming(msg const& m, node_id* id, libtorrent::dht_settings c
 	if (ret_ent == 0)
 	{
 		// it may be an error
-		ret_ent = m.message.dict_find_dict("e");
+		ret_ent = m.message.dict_find("e");
 		o->timeout();
 		if (ret_ent == NULL)
 		{

src/peer_connection.cpp

@@ -66,6 +66,8 @@ POSSIBILITY OF SUCH DAMAGE.
 #include "libtorrent/request_blocks.hpp" // for request_a_block
 #include "libtorrent/performance_counters.hpp" // for counters
 #include "libtorrent/alert_manager.hpp" // for alert_manageralert_manager
+#include "libtorrent/ip_filter.hpp"
+#include "libtorrent/kademlia/node_id.hpp"
 
 #ifdef TORRENT_DEBUG
 #include <set>
@@ -1219,6 +1221,17 @@ namespace libtorrent
 			peer_log("*** couldn't find a torrent with the given info_hash: %s torrents:", to_hex(ih.to_string()).c_str());
 			m_ses.log_all_torrents(this);
 #endif
+
+#ifndef TORRENT_DISABLE_DHT
+			if (dht::verify_random_id(ih))
+			{
+				// this means the hash was generated from our generate_random_id()
+				// as part of DHT traffic. The fact that we got an incoming
+				// connection on this info-hash, means the other end, making this
+				// connection fished it out of the DHT chatter. That's suspicious.
+				m_ses.get_ip_filter().add_rule(m_remote.address(), m_remote.address(), 0);
+			}
+#endif
 			disconnect(errors::invalid_info_hash, op_bittorrent, 1);
 			return;
 		}

src/session_impl.cpp

@@ -1130,7 +1130,7 @@ namespace aux {
 			i->second->port_filter_updated();
 	}
 
-	ip_filter const& session_impl::get_ip_filter() const
+	ip_filter& session_impl::get_ip_filter()
 	{
 		return m_ip_filter;
 	}

test/test_dht.cpp

@@ -617,6 +617,16 @@ int test_main()
 		fprintf(stderr, "   invalid get_peers response: %s\n", error_string);
 	}
 
+	// ====== test node ID testing =====
+
+	{
+		node_id rnd = generate_random_id();
+		TEST_CHECK(verify_random_id(rnd));
+
+		rnd[19] ^= 0x55;
+		TEST_CHECK(!verify_random_id(rnd));
+	}
+
 	// ====== test node ID enforcement ======
 
 	// enable node_id enforcement

tools/parse_dht_log.py

@@ -3,6 +3,9 @@ import sys
 import os
 import time
 import calendar
+import pprint
+
+pp = pprint.PrettyPrinter(indent=4)
 
 up_time_quanta = 500
 
@@ -38,6 +41,10 @@ last_incoming = ''
 
 our_node_id = ''
 
+unique_ips = set()
+client_version_histogram = {}
+client_histogram = {}
+
 for line in f:
 	counter += 1
 #	if counter % 1000 == 0:
@@ -47,6 +54,31 @@ for line in f:
 		if 'starting DHT tracker with node id:' in line:
 			our_node_id = l[l.index('id:') + 1].strip()
 
+		try:
+			if len(l) > 4 and l[2] == '<==' and l[1] == '[dht_tracker]':
+				ip = l[3].split(':')[0]
+				if ip not in unique_ips:
+					unique_ips.add(ip)
+					json_blob = line.split(l[3])[1]
+					version = json_blob.split("'v': '")[1].split("'")[0]
+					if len(version) == 4:
+						v = '%s-%d' % (version[0:2], (ord(version[2]) << 8) + ord(version[3]))
+					elif len(version) == 8:
+						v = '%c%c-%d' % (chr(int(version[0:2], 16)), chr(int(version[2:4], 16)), int(version[4:8], 16))
+					else:
+						v = 'unknown'
+
+					if not v in client_version_histogram:
+						client_version_histogram[v] = 1
+					else:
+						client_version_histogram[v] += 1
+
+					if not v[0:2] in client_histogram:
+						client_histogram[v[0:2]] = 1
+					else:
+						client_histogram[v[0:2]] += 1
+		except: pass
+
 		if 'announce-distance:' in line:
 			idx = l.index('announce-distance:')
 
@@ -215,6 +247,15 @@ for k,v in sorted(node_uptime_histogram.items()):
 	print '%f %f' % (k / float(60), s / float(total_uptime_nodes))
 out.close()
 
+
+print 'clients by version'
+client_version_histogram = sorted(client_version_histogram.items(), key=lambda x: x[1], reverse=True)
+pp.pprint(client_version_histogram)
+
+print 'clients'
+client_histogram = sorted(client_histogram.items(), key=lambda x: x[1], reverse=True)
+pp.pprint(client_histogram)
+
 out = open('dht.gnuplot', 'w+')
 out.write('''
 set term png size 1200,700 small