make dht test program able to get and put mutable items. fixed some DHT bugs along the mutable put/get path

include/libtorrent/kademlia/get_item.hpp

@@ -50,7 +50,16 @@ public:
 		boost::uint64_t seq,
 		char const* sig);
 
-	get_item(node_impl& node, node_id target, data_callback const& dcallback);
+	// for immutable itms
+	get_item(node_impl& node
+		, node_id target
+		, data_callback const& dcallback);
+
+	// for mutable items
+	get_item(node_impl& node
+		, char const* pk
+		, std::string const& salt
+		, data_callback const& dcallback);
 
 	virtual char const* name() const;
 

include/libtorrent/kademlia/item.hpp

@@ -43,20 +43,20 @@ POSSIBILITY OF SUCH DAMAGE.
 namespace libtorrent { namespace dht
 {
 
-// calculate the target hash for an item. Either v must be specified,
-// which is the content. That's for immutable items. For mutable items,
-// instead specify the salt and the public key (pk).
+// calculate the target hash for an immutable item.
 sha1_hash TORRENT_EXTRA_EXPORT item_target_id(
-	std::pair<char const*, int> v
-	, std::pair<char const*, int> salt
+	std::pair<char const*, int> v);
+
+// calculate the target hash for a mutable item.
+sha1_hash TORRENT_EXTRA_EXPORT item_target_id(std::pair<char const*, int> salt
 	, char const* pk);
 
 bool TORRENT_EXTRA_EXPORT verify_mutable_item(
-	std::pair<char const*, int> v,
-	std::pair<char const*, int> salt,
-	boost::uint64_t seq,
-	char const* pk,
-	char const* sig);
+	std::pair<char const*, int> v
+	, std::pair<char const*, int> salt
+	, boost::uint64_t seq
+	, char const* pk
+	, char const* sig);
 
 // TODO: since this is a public function, it should probably be moved
 // out of this header and into one with other public functions.
@@ -68,12 +68,12 @@ bool TORRENT_EXTRA_EXPORT verify_mutable_item(
 // is responsible for allocating the destination buffer that's passed in
 // as the ``sig`` argument. Typically it would be allocated on the stack.
 void TORRENT_EXPORT sign_mutable_item(
-	std::pair<char const*, int> v,
-	std::pair<char const*, int> salt,
-	boost::uint64_t seq,
-	char const* pk,
-	char const* sk,
-	char* sig);
+	std::pair<char const*, int> v
+	, std::pair<char const*, int> salt
+	, boost::uint64_t seq
+	, char const* pk
+	, char const* sk
+	, char* sig);
 
 sha1_hash TORRENT_EXTRA_EXPORT mutable_item_cas(
 	std::pair<char const*, int> v
@@ -96,6 +96,7 @@ class TORRENT_EXTRA_EXPORT item
 {
 public:
 	item() : m_mutable(false) {}
+	item(char const* pk, std::string const& salt);
 	item(entry const& v) { assign(v); }
 	item(entry const& v
 		, std::pair<char const*, int> salt
@@ -130,10 +131,10 @@ public:
 
 	entry const& value() const { return m_value; }
 	boost::array<char, item_pk_len> const& pk() const
-	{ TORRENT_ASSERT(m_mutable); return m_pk; }
+	{ return m_pk; }
 	boost::array<char, item_sig_len> const& sig() const
-	{ TORRENT_ASSERT(m_mutable); return m_sig; }
-	boost::uint64_t seq() const { TORRENT_ASSERT(m_mutable); return m_seq; }
+	{ return m_sig; }
+	boost::uint64_t seq() const { return m_seq; }
 	std::string const& salt() const { return m_salt; }
 
 private:

include/libtorrent/kademlia/node.hpp

@@ -237,6 +237,7 @@ public:
 		, boost::function<void(std::vector<tcp::endpoint> const&)> f);
 
 	void get_item(sha1_hash const& target, boost::function<bool(item&)> f);
+	void get_item(char const* pk, std::string const& salt, boost::function<bool(item&)> f);
 
 	bool verify_token(std::string const& token, char const* info_hash
 		, udp::endpoint const& addr);

src/kademlia/dht_tracker.cpp

@@ -472,12 +472,7 @@ namespace libtorrent { namespace dht
 		, boost::function<void(item const&)> cb
 		, std::string salt)
 	{
-		sha1_hash target = item_target_id(
-			std::pair<char const*, int>(NULL, 0)
-			, std::pair<char const*, int>(salt.c_str(), salt.size())
-			, key);
-
-		m_dht.get_item(target, boost::bind(&get_mutable_item_callback, _1, cb));
+		m_dht.get_item(key, salt, boost::bind(&get_mutable_item_callback, _1, cb));
 	}
 
 	void dht_tracker::put_item(entry data
@@ -486,8 +481,7 @@ namespace libtorrent { namespace dht
 		std::string flat_data;
 		bencode(std::back_inserter(flat_data), data);
 		sha1_hash target = item_target_id(
-			std::pair<char const*, int>(flat_data.c_str(), flat_data.size())
-			, std::pair<char const*, int>(NULL, 0), NULL);
+			std::pair<char const*, int>(flat_data.c_str(), flat_data.size()));
 
 		m_dht.get_item(target, boost::bind(&put_immutable_item_callback
 			, _1, cb, data));
@@ -496,12 +490,7 @@ namespace libtorrent { namespace dht
 	void dht_tracker::put_item(char const* key
 		, boost::function<void(item&)> cb, std::string salt)
 	{
-		sha1_hash target = item_target_id(
-			std::pair<char const*, int>(NULL, 0)
-			, std::pair<char const*, int>(salt.c_str(), salt.size())
-			, key);
-
-		m_dht.get_item(target, boost::bind(&put_mutable_item_callback
+		m_dht.get_item(key, salt, boost::bind(&put_mutable_item_callback
 			, _1, cb));
 	}
 

src/kademlia/get_item.cpp

@@ -51,7 +51,12 @@ void get_item::got_data(lazy_entry const* v,
 
 	std::pair<char const*, int> salt(m_salt.c_str(), m_salt.size());
 
-	sha1_hash incoming_target = item_target_id(v->data_section(), salt, pk);
+	sha1_hash incoming_target;
+	if (pk)
+		incoming_target = item_target_id(salt, pk);
+	else
+		incoming_target = item_target_id(v->data_section());
+
 	if (incoming_target != m_target) return;
 
 	if (pk && sig)
@@ -108,6 +113,19 @@ get_item::get_item(
 {
 }
 
+get_item::get_item(
+	node_impl& node
+	, char const* pk
+	, std::string const& salt
+	, data_callback const& dcallback)
+	: find_data(node, item_target_id(
+		std::make_pair(salt.c_str(), int(salt.size())), pk)
+		, nodes_callback())
+	, m_data_callback(dcallback)
+	, m_data(pk, salt)
+{
+}
+
 char const* get_item::name() const { return "get"; }
 
 observer_ptr get_item::new_observer(void* ptr
@@ -151,8 +169,10 @@ void get_item::done()
 #if TORRENT_USE_ASSERTS
 			if (m_data.is_mutable())
 			{
-				TORRENT_ASSERT(m_target == hasher(m_data.pk().data(),
-					item_pk_len).final());
+				TORRENT_ASSERT(m_target
+					== item_target_id(std::pair<char const*, int>(m_data.salt().c_str()
+						, m_data.salt().size())
+					, m_data.pk().data()));
 			}
 			else
 			{

src/kademlia/item.cpp

@@ -80,21 +80,21 @@ namespace
 	}
 }
 
-sha1_hash item_target_id(
-	std::pair<char const*, int> v
-	, std::pair<char const*, int> salt
+// calculate the target hash for an immutable item.
+sha1_hash item_target_id(std::pair<char const*, int> v)
+{
+	hasher h;
+	h.update(v.first, v.second);
+	return h.final();
+}
+
+// calculate the target hash for a mutable item.
+sha1_hash item_target_id(std::pair<char const*, int> salt
 	, char const* pk)
 {
 	hasher h;
-	if (pk)
-	{
-		h.update(pk, item_pk_len);
-		if (salt.second > 0) h.update(salt.first, salt.second);
-	}
-	else
-	{
-		h.update(v.first, v.second);
-	}
+	h.update(pk, item_pk_len);
+	if (salt.second > 0) h.update(salt.first, salt.second);
 	return h.final();
 }
 
@@ -160,6 +160,14 @@ sha1_hash mutable_item_cas(std::pair<char const*, int> v
 	return hasher(str, len).final();
 }
 
+item::item(char const* pk, std::string const& salt)
+	: m_mutable(true)
+	, m_salt(salt)
+	, m_seq(0)
+{
+	memcpy(m_pk.data(), pk, item_pk_len);
+}
+
 item::item(entry const& v
 	, std::pair<char const*, int> salt
 	, boost::uint64_t seq, char const* pk, char const* sk)
@@ -231,7 +239,6 @@ void item::assign(entry const& v, std::string salt, boost::uint64_t seq
 
 sha1_hash item::cas()
 {
-	TORRENT_ASSERT(m_mutable);
 	char buffer[1000];
 	int bsize = bencode(buffer, m_value);
 	return mutable_item_cas(std::make_pair(buffer, bsize)

src/kademlia/node.cpp

@@ -390,10 +390,11 @@ void node_impl::announce(sha1_hash const& info_hash, int listen_port, int flags
 	ta->start();
 }
 
-void node_impl::get_item(sha1_hash const& target, boost::function<bool(item&)> f)
+void node_impl::get_item(sha1_hash const& target
+	, boost::function<bool(item&)> f)
 {
 #ifdef TORRENT_DHT_VERBOSE_LOGGING
-	TORRENT_LOG(node) << "starting get for [ " << target << " ]" ;
+	TORRENT_LOG(node) << "starting get for [ hash: " << target << " ]" ;
 #endif
 
 	boost::intrusive_ptr<dht::get_item> ta;
@@ -401,6 +402,20 @@ void node_impl::get_item(sha1_hash const& target, boost::function<bool(item&)> f
 	ta->start();
 }
 
+void node_impl::get_item(char const* pk, std::string const& salt
+	, boost::function<bool(item&)> f)
+{
+#ifdef TORRENT_DHT_VERBOSE_LOGGING
+	TORRENT_LOG(node) << "starting get for [ key: "
+		<< to_hex(std::string(pk, 32)) << " ]" ;
+#endif
+
+	boost::intrusive_ptr<dht::get_item> ta;
+	ta.reset(new dht::get_item(*this, pk, salt, f));
+	ta->start();
+}
+
+
 void node_impl::tick()
 {
 	node_id target;
@@ -948,7 +963,11 @@ void node_impl::incoming_request(msg const& m, entry& e)
 			return;
 		}
 
-		sha1_hash target = item_target_id(buf, salt, pk);
+		sha1_hash target;
+		if (pk)
+			target = item_target_id(salt, pk);
+		else
+			target = item_target_id(buf);
 
 //		fprintf(stderr, "%s PUT target: %s salt: %s key: %s\n"
 //			, mutable_put ? "mutable":"immutable"

src/kademlia/traversal_algorithm.cpp

@@ -188,7 +188,7 @@ void traversal_algorithm::start()
 {
 	// in case the routing table is empty, use the
 	// router nodes in the table
-	if (m_results.empty()) add_router_entries();
+	if (m_results.size() < 8) add_router_entries();
 	init();
 	bool is_done = add_requests();
 	if (is_done) done();

tools/Jamfile

@@ -18,5 +18,5 @@ project tools
 
 exe parse_hash_fails : parse_hash_fails.cpp ;
 exe parse_request_log : parse_request_log.cpp ;
-exe dht : dht_put.cpp ;
+exe dht : dht_put.cpp : <include>../ed25519/src ;
 

tools/dht_put.cpp

@@ -35,6 +35,9 @@ POSSIBILITY OF SUCH DAMAGE.
 #include "libtorrent/escape_string.hpp" // for from_hex
 #include "libtorrent/alert_types.hpp"
 #include "libtorrent/bencode.hpp" // for bencode()
+#include "libtorrent/kademlia/item.hpp" // for sign_mutable_item
+#include "ed25519.h"
+#include <boost/bind.hpp>
 
 #include <stdlib.h>
 
@@ -44,10 +47,16 @@ void usage()
 {
 	fprintf(stderr,
 		"USAGE:\ndht <command> <arg>\n\nCOMMANDS:\n"
-		"get <hash>             - retrieves and prints out the immutable\n"
-		"                         item stored under hash.\n"
-		"put <string>           - puts the specified string as an immutable\n"
-		"                         item onto the DHT. The resulting target hash\n"
+		"get <hash>                - retrieves and prints out the immutable\n"
+		"                            item stored under hash.\n"
+		"put <string>              - puts the specified string as an immutable\n"
+		"                            item onto the DHT. The resulting target hash\n"
+		"gen-key <key-file>        - generate ed25519 keypair and save it in\n"
+		"                            the specified file\n"
+		"mput <key-file> <string>  - puts the specified string as a mutable\n"
+		"                            object under the public key in key-file\n"
+		"mget <public-key>         - get a mutable object under the specified\n"
+		"                            public key\n"
 		);
 	exit(1);
 }
@@ -84,8 +93,60 @@ std::auto_ptr<alert> wait_for_alert(session& s, int alert_type)
 	return ret;
 }
 
+void put_string(entry& e, boost::array<char, 64>& sig, boost::uint64_t& seq
+	, std::string const& salt, char const* public_key, char const* private_key
+	, char const* str)
+{
+	using libtorrent::dht::sign_mutable_item;
+
+	e = std::string(str);
+	std::vector<char> buf;
+	bencode(std::back_inserter(buf), e);
+	++seq;
+	sign_mutable_item(std::pair<char const*, int>(&buf[0], buf.size())
+		, std::pair<char const*, int>(&salt[0], salt.size())
+		, seq
+		, public_key
+		, private_key
+		, sig.data());
+}
+
+void bootstrap(session& s)
+{
+	printf("bootstrapping\n");
+	wait_for_alert(s, dht_bootstrap_alert::alert_type);
+}
+
 int main(int argc, char* argv[])
 {
+	// skip pointer to self
+	++argv;
+	--argc;
+
+	if (argc < 1) usage();
+
+	if (strcmp(argv[0], "gen-key") == 0)
+	{
+		++argv;
+		--argc;
+		if (argc < 1) usage();
+	
+		unsigned char seed[32];
+		ed25519_create_seed(seed);
+
+		FILE* f = fopen(argv[0], "wb+");
+		if (f == NULL)
+		{
+			fprintf(stderr, "failed to open file for writing \"%s\": (%d) %s\n"
+				, argv[0], errno, strerror(errno));
+			return 1;
+		}
+
+		fwrite(seed, 1, 32, f);
+		fclose(f);
+		return 0;
+	}
+
 	session s;
 	s.set_alert_mask(0xffffffff);
 
@@ -115,15 +176,6 @@ int main(int argc, char* argv[])
 		fclose(f);
 	}
 
-	printf("bootstrapping\n");
-	wait_for_alert(s, dht_bootstrap_alert::alert_type);
-
-	// skip pointer to self
-	++argv;
-	--argc;
-
-	if (argc < 1) usage();
-
 	if (strcmp(argv[0], "get") == 0)
 	{
 		++argv;
@@ -137,8 +189,14 @@ int main(int argc, char* argv[])
 			usage();
 		}
 		sha1_hash target;
-		from_hex(argv[0], 40, (char*)&target[0]);
+		bool ret = from_hex(argv[0], 40, (char*)&target[0]);
+		if (!ret)
+		{
+			fprintf(stderr, "invalid hex encoding of target hash\n");
+			return 1;
+		}
 
+		bootstrap(s);
 		s.dht_get_item(target);
 
 		printf("GET %s\n", to_hex(target.to_string()).c_str());
@@ -156,18 +214,87 @@ int main(int argc, char* argv[])
 	{
 		++argv;
 		--argc;
-	
 		if (argc < 1) usage();
 
 		entry data;
 		data = std::string(argv[0]);
 
+		bootstrap(s);
 		sha1_hash target = s.dht_put_item(data);
 		
 		printf("PUT %s\n", to_hex(target.to_string()).c_str());
 
 		wait_for_alert(s, dht_put_alert::alert_type);
 	}
+	else if (strcmp(argv[0], "mput") == 0)
+	{
+		++argv;
+		--argc;
+		if (argc < 1) usage();
+
+		FILE* f = fopen(argv[0], "rb+");
+		if (f == NULL)
+		{
+			fprintf(stderr, "failed to open file \"%s\": (%d) %s\n"
+				, argv[0], errno, strerror(errno));
+			return 1;
+		}
+
+		unsigned char seed[32];
+		fread(seed, 1, 32, f);
+		fclose(f);
+
+		++argv;
+		--argc;
+		if (argc < 1) usage();
+
+		boost::array<char, 32> public_key;
+		boost::array<char, 64> private_key;
+		ed25519_create_keypair((unsigned char*)public_key.data()
+			, (unsigned char*)private_key.data(), seed);
+		
+		bootstrap(s);
+		s.dht_put_item(public_key, boost::bind(&put_string, _1, _2, _3, _4
+			, public_key.data(), private_key.data(), argv[0]));
+
+		printf("public key: %s\n", to_hex(std::string(public_key.data()
+			, public_key.size())).c_str());
+
+		wait_for_alert(s, dht_put_alert::alert_type);
+		usleep(10000000);
+	}
+	else if (strcmp(argv[0], "mget") == 0)
+	{
+		++argv;
+		--argc;
+		if (argc < 1) usage();
+
+		int len = strlen(argv[0]);
+		if (len != 64)
+		{
+			fprintf(stderr, "public key is expected to be 64 hex digits\n");
+			return 1;
+		}
+		boost::array<char, 32> public_key;
+		bool ret = from_hex(argv[0], len, &public_key[0]);
+		if (!ret)
+		{
+			fprintf(stderr, "invalid hex encoding of public key\n");
+			return 1;
+		}
+
+		bootstrap(s);
+		s.dht_get_item(public_key);
+
+		std::auto_ptr<alert> a = wait_for_alert(s, dht_mutable_item_alert::alert_type);
+
+		dht_mutable_item_alert* item = alert_cast<dht_mutable_item_alert>(a.get());
+		entry data;
+		if (item)
+			data.swap(item->item);
+
+		printf("%s", data.to_string().c_str());
+	}
 	else
 	{
 		usage();