implement a sligthtly more aggressive DHT lookup mechanism

This commit is contained in:
Arvid Norberg 2013-01-20 07:54:54 +00:00
parent 002d69a9fc
commit ab7369fee9
10 changed files with 278 additions and 607 deletions

View File

@ -1,3 +1,4 @@
* improve DHT lookup speed
* improve support for windows XP and earlier
* introduce global connection priority for improved swarm performance
* make files deleted alert non-discardable

View File

@ -1433,6 +1433,7 @@ struct has the following members::
bool restrict_routing_ips;
bool restrict_search_ips;
bool extended_routing_table;
bool aggressive_lookups;
};
``max_peers_reply`` is the maximum number of peers the node will send in
@ -1474,6 +1475,12 @@ which port the DHT would listen on and send messages from. This field is depreca
and ignored. libtorrent always tries to open the UDP socket on the same port
as the TCP socket.
``aggressive_lookups`` slightly changes the lookup behavior in terms of how
many outstanding requests we keep. Instead of having branch factor be a hard
limit, we always keep *branch factor* outstanding requests to the closest nodes.
i.e. every time we get results back with closer nodes, we query them right away.
It lowers the lookup times at the cost of more outstanding queries.
``is_dht_running()`` returns true if the DHT support has been started and false
otherwise.

View File

@ -1,540 +0,0 @@
<?xml version="1.0" encoding="utf-8" ?>
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
<head>
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
<meta name="generator" content="Docutils 0.5: http://docutils.sourceforge.net/" />
<title>Bittorrent udp-tracker protocol extension</title>
<meta name="author" content="Arvid Norberg, arvid&#64;rasterbar.com" />
<link rel="stylesheet" type="text/css" href="../../css/base.css" />
<link rel="stylesheet" type="text/css" href="../../css/rst.css" />
<script type="text/javascript">
/* <![CDATA[ */
(function() {
var s = document.createElement('script'), t = document.getElementsByTagName('script')[0];
s.type = 'text/javascript';
s.async = true;
s.src = 'http://api.flattr.com/js/0.6/load.js?mode=auto';
t.parentNode.insertBefore(s, t);
})();
/* ]]> */
</script>
<link rel="stylesheet" href="style.css" type="text/css" />
<style type="text/css">
/* Hides from IE-mac \*/
* html pre { height: 1%; }
/* End hide from IE-mac */
</style>
</head>
<body>
<div class="document" id="bittorrent-udp-tracker-protocol-extension">
<div id="container">
<div id="headerNav">
<ul>
<li class="first"><a href="/">Home</a></li>
<li><a href="../../products.html">Products</a></li>
<li><a href="../../contact.html">Contact</a></li>
</ul>
</div>
<div id="header">
<h1><span>Rasterbar Software</span></h1>
<h2><span>Software developement and consulting</span></h2>
</div>
<div id="main">
<h1 class="title">Bittorrent udp-tracker protocol extension</h1>
<table class="docinfo" frame="void" rules="none">
<col class="docinfo-name" />
<col class="docinfo-content" />
<tbody valign="top">
<tr><th class="docinfo-name">Author:</th>
<td>Arvid Norberg, <a class="last reference external" href="mailto:arvid&#64;rasterbar.com">arvid&#64;rasterbar.com</a></td></tr>
</tbody>
</table>
<div class="contents topic" id="table-of-contents">
<p class="topic-title first">Table of contents</p>
<ul class="simple">
<li><a class="reference internal" href="#introduction" id="id2">introduction</a></li>
<li><a class="reference internal" href="#connecting" id="id3">connecting</a></li>
<li><a class="reference internal" href="#announcing" id="id4">announcing</a></li>
<li><a class="reference internal" href="#scraping" id="id5">scraping</a></li>
<li><a class="reference internal" href="#errors" id="id6">errors</a></li>
<li><a class="reference internal" href="#actions" id="id7">actions</a></li>
<li><a class="reference internal" href="#extensions" id="id8">extensions</a><ul>
<li><a class="reference internal" href="#authentication" id="id9">authentication</a></li>
</ul>
</li>
<li><a class="reference internal" href="#credits" id="id10">credits</a></li>
</ul>
</div>
<div class="section" id="introduction">
<h1>introduction</h1>
<p>A tracker with the protocol &quot;udp://&quot; in its URI
is supposed to be contacted using this protocol.</p>
<p>This protocol is supported by
<a class="reference external" href="http://xbtt.sourceforge.net">xbt-tracker</a>.</p>
<p>For additional information and descritptions of
the terminology used in this document, see
the <a class="reference external" href="http://wiki.theory.org/index.php/BitTorrentSpecification">protocol specification</a></p>
<p>All values are sent in network byte order (big endian). The sizes
are specified with ANSI-C standard types.</p>
<p>If no response to a request is received within 15 seconds, resend
the request. If no reply has been received after 60 seconds, stop
retrying.</p>
</div>
<div class="section" id="connecting">
<h1>connecting</h1>
<p>Client sends packet:</p>
<table border="1" class="docutils">
<colgroup>
<col width="18%" />
<col width="28%" />
<col width="54%" />
</colgroup>
<thead valign="bottom">
<tr><th class="head">size</th>
<th class="head">name</th>
<th class="head">description</th>
</tr>
</thead>
<tbody valign="top">
<tr><td>int64_t</td>
<td>connection_id</td>
<td>Must be initialized to 0x41727101980
in network byte order. This will
identify the protocol.</td>
</tr>
<tr><td>int32_t</td>
<td>action</td>
<td>0 for a connection request</td>
</tr>
<tr><td>int32_t</td>
<td>transaction_id</td>
<td>Randomized by client.</td>
</tr>
</tbody>
</table>
<p>Server replies with packet:</p>
<table border="1" class="docutils">
<colgroup>
<col width="18%" />
<col width="28%" />
<col width="54%" />
</colgroup>
<thead valign="bottom">
<tr><th class="head">size</th>
<th class="head">name</th>
<th class="head">description</th>
</tr>
</thead>
<tbody valign="top">
<tr><td>int32_t</td>
<td>action</td>
<td>Describes the type of packet, in this
case it should be 0, for connect.
If 3 (for error) see <a class="reference internal" href="#errors">errors</a>.</td>
</tr>
<tr><td>int32_t</td>
<td>transaction_id</td>
<td>Must match the transaction_id sent
from the client.</td>
</tr>
<tr><td>int64_t</td>
<td>connection_id</td>
<td>A connection id, this is used when
further information is exchanged with
the tracker, to identify you.
This connection id can be reused for
multiple requests, but if it's cached
for too long, it will not be valid
anymore.</td>
</tr>
</tbody>
</table>
</div>
<div class="section" id="announcing">
<h1>announcing</h1>
<p>Client sends packet:</p>
<table border="1" class="docutils">
<colgroup>
<col width="18%" />
<col width="28%" />
<col width="54%" />
</colgroup>
<thead valign="bottom">
<tr><th class="head">size</th>
<th class="head">name</th>
<th class="head">description</th>
</tr>
</thead>
<tbody valign="top">
<tr><td>int64_t</td>
<td>connection_id</td>
<td>The connection id acquired from
establishing the connection.</td>
</tr>
<tr><td>int32_t</td>
<td>action</td>
<td>Action. in this case, 1 for announce.
See <a class="reference internal" href="#actions">actions</a>.</td>
</tr>
<tr><td>int32_t</td>
<td>transaction_id</td>
<td>Randomized by client.</td>
</tr>
<tr><td>int8_t[20]</td>
<td>info_hash</td>
<td>The info-hash of the torrent you want
announce yourself in.</td>
</tr>
<tr><td>int8_t[20]</td>
<td>peer_id</td>
<td>Your peer id.</td>
</tr>
<tr><td>int64_t</td>
<td>downloaded</td>
<td>The number of byte you've downloaded
in this session.</td>
</tr>
<tr><td>int64_t</td>
<td>left</td>
<td>The number of bytes you have left to
download until you're finished.</td>
</tr>
<tr><td>int64_t</td>
<td>uploaded</td>
<td>The number of bytes you have uploaded
in this session.</td>
</tr>
<tr><td>int32_t</td>
<td>event</td>
<td><p class="first">The event, one of</p>
<blockquote class="last">
<ul class="simple">
<li>none = 0</li>
<li>completed = 1</li>
<li>started = 2</li>
<li>stopped = 3</li>
</ul>
</blockquote>
</td>
</tr>
<tr><td>uint32_t</td>
<td>ip</td>
<td>Your ip address. Set to 0 if you want
the tracker to use the <tt class="docutils literal"><span class="pre">sender</span></tt> of
this udp packet.</td>
</tr>
<tr><td>uint32_t</td>
<td>key</td>
<td>A unique key that is randomized by the
client.</td>
</tr>
<tr><td>int32_t</td>
<td>num_want</td>
<td>The maximum number of peers you want
in the reply. Use -1 for default.</td>
</tr>
<tr><td>uint16_t</td>
<td>port</td>
<td>The port you're listening on.</td>
</tr>
<tr><td>uint16_t</td>
<td>extensions</td>
<td>See <a class="reference internal" href="#extensions">extensions</a></td>
</tr>
</tbody>
</table>
<p>Server replies with packet:</p>
<table border="1" class="docutils">
<colgroup>
<col width="18%" />
<col width="28%" />
<col width="54%" />
</colgroup>
<thead valign="bottom">
<tr><th class="head">size</th>
<th class="head">name</th>
<th class="head">description</th>
</tr>
</thead>
<tbody valign="top">
<tr><td>int32_t</td>
<td>action</td>
<td>The action this is a reply to. Should
in this case be 1 for announce.
If 3 (for error) see <a class="reference internal" href="#errors">errors</a>.
See <a class="reference internal" href="#actions">actions</a>.</td>
</tr>
<tr><td>int32_t</td>
<td>transaction_id</td>
<td>Must match the transaction_id sent
in the announce request.</td>
</tr>
<tr><td>int32_t</td>
<td>interval</td>
<td>the number of seconds you should wait
until reannouncing yourself.</td>
</tr>
<tr><td>int32_t</td>
<td>leechers</td>
<td>The number of peers in the swarm that
has not finished downloading.</td>
</tr>
<tr><td>int32_t</td>
<td>seeders</td>
<td>The number of peers in the swarm that
has finished downloading and are
seeding.</td>
</tr>
</tbody>
</table>
<p>The rest of the server reply is a variable number of the following structure:</p>
<table border="1" class="docutils">
<colgroup>
<col width="18%" />
<col width="28%" />
<col width="54%" />
</colgroup>
<thead valign="bottom">
<tr><th class="head">size</th>
<th class="head">name</th>
<th class="head">description</th>
</tr>
</thead>
<tbody valign="top">
<tr><td>int32_t</td>
<td>ip</td>
<td>The ip of a peer in the swarm.</td>
</tr>
<tr><td>uint16_t</td>
<td>port</td>
<td>The peer's listen port.</td>
</tr>
</tbody>
</table>
</div>
<div class="section" id="scraping">
<h1>scraping</h1>
<p>Client sends packet:</p>
<table border="1" class="docutils">
<colgroup>
<col width="18%" />
<col width="28%" />
<col width="54%" />
</colgroup>
<thead valign="bottom">
<tr><th class="head">size</th>
<th class="head">name</th>
<th class="head">description</th>
</tr>
</thead>
<tbody valign="top">
<tr><td>int64_t</td>
<td>connection_id</td>
<td>The connection id retreived from the
establishing of the connection.</td>
</tr>
<tr><td>int32_t</td>
<td>action</td>
<td>The action, in this case, 2 for
scrape. See <a class="reference internal" href="#actions">actions</a>.</td>
</tr>
<tr><td>int32_t</td>
<td>transaction_id</td>
<td>Randomized by client.</td>
</tr>
</tbody>
</table>
<p>The following structure is repeated for each info-hash to scrape, but limited by
the MTU.</p>
<table border="1" class="docutils">
<colgroup>
<col width="18%" />
<col width="28%" />
<col width="54%" />
</colgroup>
<thead valign="bottom">
<tr><th class="head">size</th>
<th class="head">name</th>
<th class="head">description</th>
</tr>
</thead>
<tbody valign="top">
<tr><td>int8_t[20]</td>
<td>info_hash</td>
<td>The info hash that is to be scraped.</td>
</tr>
</tbody>
</table>
<p>Server replies with packet:</p>
<table border="1" class="docutils">
<colgroup>
<col width="18%" />
<col width="28%" />
<col width="54%" />
</colgroup>
<thead valign="bottom">
<tr><th class="head">size</th>
<th class="head">name</th>
<th class="head">description</th>
</tr>
</thead>
<tbody valign="top">
<tr><td>int32_t</td>
<td>action</td>
<td>The action, should in this case be
2 for scrape.
If 3 (for error) see <a class="reference internal" href="#errors">errors</a>.</td>
</tr>
<tr><td>int32_t</td>
<td>transaction_id</td>
<td>Must match the sent transaction id.</td>
</tr>
</tbody>
</table>
<p>The rest of the packet contains the following structures once for each info-hash
you asked in the scrape request.</p>
<table border="1" class="docutils">
<colgroup>
<col width="18%" />
<col width="28%" />
<col width="54%" />
</colgroup>
<thead valign="bottom">
<tr><th class="head">size</th>
<th class="head">name</th>
<th class="head">description</th>
</tr>
</thead>
<tbody valign="top">
<tr><td>int32_t</td>
<td>complete</td>
<td>The current number of connected seeds.</td>
</tr>
<tr><td>int32_t</td>
<td>downloaded</td>
<td>The number of times this torrent has
been downloaded.</td>
</tr>
<tr><td>int32_t</td>
<td>incomplete</td>
<td>The current number of connected
leechers.</td>
</tr>
</tbody>
</table>
</div>
<div class="section" id="errors">
<h1>errors</h1>
<p>In case of a tracker error,</p>
<p>server replies packet:</p>
<table border="1" class="docutils">
<colgroup>
<col width="18%" />
<col width="28%" />
<col width="54%" />
</colgroup>
<thead valign="bottom">
<tr><th class="head">size</th>
<th class="head">name</th>
<th class="head">description</th>
</tr>
</thead>
<tbody valign="top">
<tr><td>int32_t</td>
<td>action</td>
<td>The action, in this case 3, for error.
See <a class="reference internal" href="#actions">actions</a>.</td>
</tr>
<tr><td>int32_t</td>
<td>transaction_id</td>
<td>Must match the transaction_id sent
from the client.</td>
</tr>
<tr><td>int8_t[]</td>
<td>error_string</td>
<td>The rest of the packet is a string
describing the error.</td>
</tr>
</tbody>
</table>
</div>
<div class="section" id="actions">
<h1>actions</h1>
<p>The action fields has the following encoding:</p>
<blockquote>
<ul class="simple">
<li>connect = 0</li>
<li>announce = 1</li>
<li>scrape = 2</li>
<li>error = 3 (only in server replies)</li>
</ul>
</blockquote>
</div>
<div class="section" id="extensions">
<h1>extensions</h1>
<p>The extensions field is a bitmask. The following
bits are assigned:</p>
<blockquote>
<ul class="simple">
<li>1 = <a class="reference internal" href="#authentication">authentication</a>.</li>
</ul>
</blockquote>
<div class="section" id="authentication">
<h2>authentication</h2>
<p>The packet will have an authentication part
appended to it. It has the following format:</p>
<table border="1" class="docutils">
<colgroup>
<col width="18%" />
<col width="28%" />
<col width="54%" />
</colgroup>
<thead valign="bottom">
<tr><th class="head">size</th>
<th class="head">name</th>
<th class="head">description</th>
</tr>
</thead>
<tbody valign="top">
<tr><td>int8_t</td>
<td>username_length</td>
<td>The number of characters in the
username.</td>
</tr>
<tr><td>int8_t[]</td>
<td>username</td>
<td>The username, the number of characters
as specified in the previous field.</td>
</tr>
<tr><td>uint8_t[8]</td>
<td>passwd_hash</td>
<td>sha1(packet + sha1(password))
The packet in this case means the
entire packet except these 8 bytes
that are the password hash. These are
the 8 first bytes (most significant)
from the 20 bytes hash calculated.</td>
</tr>
</tbody>
</table>
</div>
</div>
<div class="section" id="credits">
<h1>credits</h1>
<p>Protocol designed by Olaf van der Spek</p>
</div>
</div>
<div id="footer">
<span>Copyright &copy; 2005 Rasterbar Software.</span>
</div>
</div>
<script src="http://www.google-analytics.com/urchin.js" type="text/javascript">
</script>
<script type="text/javascript">
_uacct = "UA-1599045-1";
urchinTracker();
</script>
</div>
</body>
</html>

View File

@ -79,10 +79,14 @@ struct traversal_algorithm : boost::noncopyable
void add_entry(node_id const& id, udp::endpoint addr, unsigned char flags);
traversal_algorithm(node_impl& node, node_id target);
int invoke_count() const { return m_invoke_count; }
int branch_factor() const { return m_branch_factor; }
protected:
void add_requests();
// returns true if we're done
bool add_requests();
void add_router_entries();
void init();

View File

@ -959,6 +959,7 @@ namespace libtorrent
, restrict_routing_ips(true)
, restrict_search_ips(true)
, extended_routing_table(true)
, aggressive_lookups(true)
{}
// the maximum number of peers to send in a
@ -1005,6 +1006,11 @@ namespace libtorrent
// table are enlarged, to make room for more nodes in order
// to lower the look-up times
bool extended_routing_table;
// makes lookups waste less time finding results,
// at the cost of being more likely to keep more
// outstanding requests
bool aggressive_lookups;
};
#endif

View File

@ -1,6 +1,8 @@
#! /usr/bin/env python
import sys
import os
import time
import calendar
up_time_quanta = 2000
@ -11,20 +13,34 @@ node_uptime_histogram = {}
counter = 0;
# maps search_id to a list of events. Each event is a dict containing:
# t: timestamp
# d: distance (from target)
# o: outstanding searches
# e: event (NEW, COMPLETED, ADD, INVOKE, TIMEOUT)
outstanding_searches = {}
# list of completed searches
searches = []
def convert_timestamp(t):
parts = t.split('.')
posix = time.strptime(parts[0], '%H:%M:%S')
return (posix.tm_hour * 3600 + posix.tm_min * 60 + posix.tm_sec) * 1000 + int(parts[1])
for line in f:
counter += 1
# if counter % 1000 == 0:
# print '\r%d' % counter,
try:
if 'distance:' in line:
l = line.split(' ')
idx = l.index('distance:')
l = line.split(' ')
if 'announce-distance:' in line:
idx = l.index('announce-distance:')
d = int(l[idx+1].strip())
if not d in announce_histogram: announce_histogram[d] = 0
announce_histogram[d] += 1
if 'NODE FAILED' in line:
l = line.split(' ')
idx = l.index('fails:')
if int(l[idx+1].strip()) != 1: continue;
idx = l.index('up-time:')
@ -33,9 +49,77 @@ for line in f:
d = d - (d % up_time_quanta)
if not d in node_uptime_histogram: node_uptime_histogram[d] = 0
node_uptime_histogram[d] += 1
search_id = l[2]
ts = l[0]
event = l[3]
if event == 'NEW':
outstanding_searches[search_id] = [{ 't': ts, 'd': 160, 'o': 0, 'e': 'NEW'}]
elif event == 'INVOKE' or event == 'ADD' or event == '1ST_TIMEOUT' or event == 'TIMEOUT' or event == 'PEERS':
if not search_id in outstanding_searches:
print 'orphaned event: %s' % line
else:
outstanding = int(l[l.index('invoke-count:')+1])
distance = int(l[l.index('distance:')+1])
outstanding_searches[search_id].append({ 't': ts, 'd': distance, 'o': outstanding + 1, 'e': event})
elif event == 'COMPLETED':
distance = int(l[l.index('distance:')+1])
outstanding_searches[search_id].append({ 't': ts, 'd': distance, 'o': 0, 'e': event})
s = outstanding_searches[search_id]
try:
start_time = convert_timestamp(s[0]['t'])
for i in range(len(s)):
s[i]['t'] = convert_timestamp(s[i]['t']) - start_time
except:
pass
searches.append(s)
del outstanding_searches[search_id]
except Exception, e:
print line.split(' ')
lookup_times_min = []
lookup_times_max = []
for s in searches:
for i in s:
if i['e'] != 'PEERS': continue
lookup_times_min.append(i['t'])
break
for i in reversed(s):
if i['e'] != 'PEERS': continue
lookup_times_max.append(i['t'])
break
lookup_times_min.sort()
lookup_times_max.sort()
out = open('dht_lookup_times_cdf.txt', 'w+')
counter = 0
for i in range(len(lookup_times_min)):
counter += 1
print >>out, '%d\t%d\t%f' % (lookup_times_min[i], lookup_times_max[i], counter / float(len(lookup_times_min)))
out.close()
out = open('dht_lookups.txt', 'w+')
for s in searches:
for i in s:
if i['e'] == 'INVOKE':
print >>out, ' ->', i['t'], i['d']
elif i['e'] == '1ST_TIMEOUT':
print >>out, ' x ', i['t'], i['d']
elif i['e'] == 'TIMEOUT':
print >>out, ' X ', i['t'], i['d']
elif i['e'] == 'PEERS':
print >>out, ' <-', i['t'], i['d']
elif i['e'] == 'COMPLETED':
print >>out, '***', i['t'], i['d'], '\n'
break
out.close()
out = open('dht_announce_distribution.dat', 'w+')
print 'announce distribution items: %d' % len(announce_histogram)
for k,v in announce_histogram.items():
@ -51,8 +135,44 @@ out.close()
out = open('dht.gnuplot', 'w+')
out.write('''
set term png size 1200,700 small
set output "dht_lookup_times_cdf.png"
set title "portion of lookups that have received at least one data response"
set ylabel "portion of lookups"
set xlabel "time from start of lookup (ms)"
set grid
plot "dht_lookup_times_cdf.txt" using 1:3 with lines title "time to first result", \
"dht_lookup_times_cdf.txt" using 2:3 with lines title "time to last result"
set terminal postscript
set output "dht_lookup_times_cdf.ps"
replot
set term png size 1200,700 small
set xtics 100
set xrange [0:2000]
set output "dht_min_lookup_times_cdf.png"
plot "dht_lookup_times_cdf.txt" using 1:3 with lines title "time to first result"
set terminal postscript
set output "dht_min_lookup_times_cdf.ps"
replot
set term png size 1200,700 small
set output "dht_node_uptime_distribution.png"
set xrange [*:*]
set title "node up time"
set ylabel "# of nodes"
set xlabel "uptime (seconds)"
set xtics auto
unset grid
set boxwidth %f
set style fill solid border -1 pattern 2
plot "dht_node_uptime_distribution.dat" using 1:2 title "nodes" with boxes
set term png size 1200,700 small
set output "dht_announce_distribution.png"
set xrange [0:*]
set title "bucket # announces are made against relative to target node-id"
set ylabel "# of announces"
set style fill solid border -1 pattern 2
@ -62,15 +182,9 @@ set terminal postscript
set output "dht_announce_distribution.ps"
replot
set term png size 1200,700 small
set output "dht_node_uptime_distribution.png"
set title "node up time"
set ylabel "# of nodes"
set xlabel "uptime (seconds)"
set boxwidth %f
set style fill solid border -1 pattern 2
plot "dht_node_uptime_distribution.dat" using 1:2 title "nodes" with boxes
''' % up_time_quanta)
out.close()
os.system('gnuplot dht.gnuplot');

View File

@ -234,7 +234,7 @@ namespace libtorrent { namespace dht
rpc_log().enable(false);
node_log().enable(false);
traversal_log().enable(false);
// traversal_log().enable(false);
// dht_tracker_log.enable(false);
TORRENT_LOG(dht_tracker) << "starting DHT tracker with node id: " << m_dht.nid();

View File

@ -56,11 +56,6 @@ using detail::read_v6_endpoint;
void find_data_observer::reply(msg const& m)
{
#ifdef TORRENT_DHT_VERBOSE_LOGGING
std::stringstream log_line;
log_line << "[" << m_algorithm.get() << "] incoming get_peer response [ ";
#endif
lazy_entry const* r = m.message.dict_find_dict("r");
if (!r)
{
@ -78,16 +73,11 @@ void find_data_observer::reply(msg const& m)
#endif
return;
}
lazy_entry const* token = r->dict_find_string("token");
if (token)
{
static_cast<find_data*>(m_algorithm.get())->got_write_token(
node_id(id->string_ptr()), token->string_value());
#ifdef TORRENT_DHT_VERBOSE_LOGGING
log_line << " token: " << to_hex(token->string_value());
#endif
}
// look for peers
@ -102,7 +92,14 @@ void find_data_observer::reply(msg const& m)
char const* end = peers + n->list_at(0)->string_length();
#ifdef TORRENT_DHT_VERBOSE_LOGGING
log_line << " p: " << ((end - peers) / 6);
TORRENT_LOG(traversal)
<< "[" << m_algorithm.get() << "] PEERS"
<< " invoke-count: " << m_algorithm->invoke_count()
<< " branch-factor: " << m_algorithm->branch_factor()
<< " addr: " << m.addr
<< " id: " << node_id(id->string_ptr())
<< " distance: " << distance_exp(m_algorithm->target(), node_id(id->string_ptr()))
<< " p: " << ((end - peers) / 6);
#endif
while (end - peers >= 6)
peer_list.push_back(read_v4_endpoint<tcp::endpoint>(peers));
@ -112,7 +109,14 @@ void find_data_observer::reply(msg const& m)
// assume it's uTorrent/libtorrent format
read_endpoint_list<tcp::endpoint>(n, peer_list);
#ifdef TORRENT_DHT_VERBOSE_LOGGING
log_line << " p: " << n->list_size();
TORRENT_LOG(traversal)
<< "[" << m_algorithm.get() << "] PEERS"
<< " invoke-count: " << m_algorithm->invoke_count()
<< " branch-factor: " << m_algorithm->branch_factor()
<< " addr: " << m.addr
<< " id: " << node_id(id->string_ptr())
<< " distance: " << distance_exp(m_algorithm->target(), node_id(id->string_ptr()))
<< " p: " << n->list_size();
#endif
}
static_cast<find_data*>(m_algorithm.get())->got_peers(peer_list);
@ -126,9 +130,6 @@ void find_data_observer::reply(msg const& m)
char const* nodes = n->string_ptr();
char const* end = nodes + n->string_length();
#ifdef TORRENT_DHT_VERBOSE_LOGGING
log_line << " nodes: " << ((end - nodes) / 26);
#endif
while (end - nodes >= 26)
{
node_id id;
@ -141,9 +142,6 @@ void find_data_observer::reply(msg const& m)
n = r->dict_find_list("nodes2");
if (n)
{
#ifdef TORRENT_DHT_VERBOSE_LOGGING
log_line << " nodes2: " << n->list_size();
#endif
for (int i = 0; i < n->list_size(); ++i)
{
lazy_entry const* p = n->list_at(0);
@ -162,10 +160,6 @@ void find_data_observer::reply(msg const& m)
#endif
}
}
#ifdef TORRENT_DHT_VERBOSE_LOGGING
log_line << " ]";
TORRENT_LOG(traversal) << log_line.str();
#endif
done();
}
@ -232,7 +226,7 @@ void find_data::done()
m_done = true;
#ifdef TORRENT_DHT_VERBOSE_LOGGING
TORRENT_LOG(traversal) << time_now_string() << "[" << this << "] get_peers DONE";
TORRENT_LOG(traversal) << "[" << this << "] get_peers DONE";
#endif
std::vector<std::pair<node_entry, std::string> > results;

View File

@ -266,7 +266,7 @@ namespace
, end(v.end()); i != end; ++i)
{
#ifdef TORRENT_DHT_VERBOSE_LOGGING
TORRENT_LOG(node) << " distance: " << (160 - distance_exp(ih, i->first.id));
TORRENT_LOG(node) << " announce-distance: " << (160 - distance_exp(ih, i->first.id));
#endif
void* ptr = node.m_rpc.allocate_observer();
@ -666,7 +666,10 @@ void node_impl::incoming_request(msg const& m, entry& e)
if (msg_keys[3] && msg_keys[3]->int_value() != 0) scrape = true;
lookup_peers(info_hash, prefix, reply, noseed, scrape);
#ifdef TORRENT_DHT_VERBOSE_LOGGING
if (reply.find_key("values")) TORRENT_LOG(node) << " values: " << reply["values"].list().size();
if (reply.find_key("values"))
{
TORRENT_LOG(node) << " values: " << reply["values"].list().size();
}
#endif
}
else if (strcmp(query, "find_node") == 0)

View File

@ -69,10 +69,11 @@ traversal_algorithm::traversal_algorithm(
, m_branch_factor(3)
, m_responses(0)
, m_timeouts(0)
, m_num_target_nodes(m_node.m_table.bucket_size() * 2)
, m_num_target_nodes(m_node.m_table.bucket_size())
{
#ifdef TORRENT_DHT_VERBOSE_LOGGING
TORRENT_LOG(traversal) << " [" << this << "] new traversal process. Target: " << target;
TORRENT_LOG(traversal) << "[" << this << "] NEW"
" target: " << target << " k: " << m_node.m_table.bucket_size();
#endif
}
@ -98,8 +99,7 @@ void traversal_algorithm::add_entry(node_id const& id, udp::endpoint addr, unsig
if (ptr == 0)
{
#ifdef TORRENT_DHT_VERBOSE_LOGGING
TORRENT_LOG(traversal) << "[" << this << ":" << name()
<< "] failed to allocate memory for observer. aborting!";
TORRENT_LOG(traversal) << "[" << this << "] failed to allocate memory for observer. aborting!";
#endif
done();
return;
@ -139,19 +139,24 @@ void traversal_algorithm::add_entry(node_id const& id, udp::endpoint addr, unsig
// close to this one. We know that it's not the same, because
// it claims a different node-ID. Ignore this to avoid attacks
#ifdef TORRENT_DHT_VERBOSE_LOGGING
TORRENT_LOG(traversal) << "ignoring DHT search entry: " << o->id()
<< " " << o->target_addr()
TORRENT_LOG(traversal) << "[" << this << "] IGNORING result "
<< "id: " << o->id()
<< " address: " << o->target_addr()
<< " existing node: "
<< (*j)->id() << " " << (*j)->target_addr();
<< (*j)->id() << " " << (*j)->target_addr()
<< " distance: " << distance_exp(m_target, o->id());
#endif
return;
}
}
TORRENT_ASSERT(std::find_if(m_results.begin(), m_results.end()
, boost::bind(&observer::id, _1) == id) == m_results.end());
#ifdef TORRENT_DHT_VERBOSE_LOGGING
TORRENT_LOG(traversal) << "[" << this << ":" << name()
<< "] adding result: " << id << " " << addr;
TORRENT_LOG(traversal) << "[" << this << "] ADD id: " << id
<< " address: " << addr
<< " distance: " << distance_exp(m_target, id)
<< " invoke-count: " << m_invoke_count;
#endif
i = m_results.insert(i, o);
}
@ -189,8 +194,9 @@ void traversal_algorithm::traverse(node_id const& id, udp::endpoint addr)
{
#ifdef TORRENT_DHT_VERBOSE_LOGGING
if (id.is_all_zeros())
TORRENT_LOG(traversal) << time_now_string() << "[" << this << ":" << name()
<< "] WARNING: node returned a list which included a node with id 0";
{
TORRENT_LOG(traversal) << time_now_string() << "[" << this << "] WARNING node returned a list which included a node with id 0";
}
#endif
add_entry(id, addr, 0);
}
@ -215,8 +221,8 @@ void traversal_algorithm::finished(observer_ptr o)
++m_responses;
--m_invoke_count;
TORRENT_ASSERT(m_invoke_count >= 0);
add_requests();
if (m_invoke_count == 0) done();
bool is_done = add_requests();
if (is_done) done();
}
// prevent request means that the total number of requests has
@ -241,9 +247,10 @@ void traversal_algorithm::failed(observer_ptr o, int flags)
++m_branch_factor;
o->flags |= observer::flag_short_timeout;
#ifdef TORRENT_DHT_VERBOSE_LOGGING
TORRENT_LOG(traversal) << " [" << this << ":" << name()
<< "] first chance timeout: "
<< o->id() << " " << o->target_ep()
TORRENT_LOG(traversal) << "[" << this << "] 1ST_TIMEOUT "
<< " id: " << o->id()
<< " distance: " << distance_exp(m_target, o->id())
<< " addr: " << o->target_ep()
<< " branch-factor: " << m_branch_factor
<< " invoke-count: " << m_invoke_count;
#endif
@ -257,8 +264,10 @@ void traversal_algorithm::failed(observer_ptr o, int flags)
--m_branch_factor;
#ifdef TORRENT_DHT_VERBOSE_LOGGING
TORRENT_LOG(traversal) << " [" << this << ":" << name()
<< "] failed: " << o->id() << " " << o->target_ep()
TORRENT_LOG(traversal) << "[" << this << "] TIMEOUT "
<< " id: " << o->id()
<< " distance: " << distance_exp(m_target, o->id())
<< " addr: " << o->target_ep()
<< " branch-factor: " << m_branch_factor
<< " invoke-count: " << m_invoke_count;
#endif
@ -276,49 +285,122 @@ void traversal_algorithm::failed(observer_ptr o, int flags)
--m_branch_factor;
if (m_branch_factor <= 0) m_branch_factor = 1;
}
add_requests();
if (m_invoke_count == 0) done();
bool is_done = add_requests();
if (is_done) done();
}
void traversal_algorithm::done()
{
#ifdef TORRENT_DHT_VERBOSE_LOGGING
int results_target = m_num_target_nodes;
int closest_target = 160;
for (std::vector<observer_ptr>::iterator i = m_results.begin()
, end(m_results.end()); i != end && results_target > 0; ++i)
{
boost::intrusive_ptr<observer> o = *i;
if (o->flags & observer::flag_alive)
{
TORRENT_ASSERT(o->flags & observer::flag_queried);
TORRENT_LOG(traversal) << "[" << this << "] "
<< results_target
<< " id: " << o->id()
<< " distance: " << distance_exp(m_target, o->id())
<< " address: " << o->target_ep();
--results_target;
int dist = distance_exp(m_target, o->id());
if (dist < closest_target) closest_target = dist;
}
}
TORRENT_LOG(traversal) << "[" << this << "] COMPLETED "
<< "distance: " << closest_target;
#endif
// delete all our references to the observer objects so
// they will in turn release the traversal algorithm
m_results.clear();
}
void traversal_algorithm::add_requests()
bool traversal_algorithm::add_requests()
{
int results_target = m_num_target_nodes;
// this only counts outstanding requests at the top of the
// target list. This is <= m_invoke count. m_invoke_count
// is the total number of outstanding requests, including
// old ones that may be waiting on nodes much farther behind
// the current point we've reached in the search.
int outstanding = 0;
// if we're doing aggressive lookups, we keep branch-factor
// outstanding requests _at the tops_ of the result list. Otherwise
// we just keep any branch-factor outstanding requests
bool agg = m_node.settings().aggressive_lookups;
// Find the first node that hasn't already been queried.
// and make sure that the 'm_branch_factor' top nodes
// stay queried at all times (obviously ignoring failed nodes)
// and without surpassing the 'result_target' nodes (i.e. k=8)
// this is a slight variation of the original paper which instead
// limits the number of outstanding requests, this limits the
// number of good outstanding requests. It will use more traffic,
// but is intended to speed up lookups
for (std::vector<observer_ptr>::iterator i = m_results.begin()
, end(m_results.end()); i != end
&& results_target > 0 && m_invoke_count < m_branch_factor; ++i)
&& results_target > 0
&& (agg ? outstanding < m_branch_factor
: m_invoke_count < m_branch_factor);
++i)
{
if ((*i)->flags & observer::flag_alive) --results_target;
if ((*i)->flags & observer::flag_queried) continue;
observer* o = i->get();
if (o->flags & observer::flag_alive)
{
TORRENT_ASSERT(o->flags & observer::flag_queried);
--results_target;
continue;
}
if (o->flags & observer::flag_queried)
{
// if it's queried, not alive and not failed, it
// must be currently in flight
if ((o->flags & observer::flag_failed) == 0)
++outstanding;
continue;
}
#ifdef TORRENT_DHT_VERBOSE_LOGGING
TORRENT_LOG(traversal) << " [" << this << ":" << name() << "]"
TORRENT_LOG(traversal) << "[" << this << "] INVOKE "
<< " nodes-left: " << (m_results.end() - i)
<< " top-invoke-count: " << outstanding
<< " invoke-count: " << m_invoke_count
<< " branch-factor: " << m_branch_factor;
<< " branch-factor: " << m_branch_factor
<< " distance: " << distance_exp(m_target, (*i)->id())
;
#endif
if (invoke(*i))
{
TORRENT_ASSERT(m_invoke_count >= 0);
++m_invoke_count;
(*i)->flags |= observer::flag_queried;
o->flags |= observer::flag_queried;
++outstanding;
}
}
// this is the completion condition. If we found m_num_target_nodes
// (i.e. k=8) completed results, without finding any still
// outstanding requests, we're done.
// also, if invoke count is 0, it means we didn't even find 'k'
// working nodes, we still have to terminate though.
return (results_target == 0 && outstanding == 0) || m_invoke_count == 0;
}
void traversal_algorithm::add_router_entries()
{
#ifdef TORRENT_DHT_VERBOSE_LOGGING
TORRENT_LOG(traversal) << " using router nodes to initiate traversal algorithm. "
TORRENT_LOG(traversal) << "[" << this << "] using router nodes to initiate traversal algorithm. "
<< std::distance(m_node.m_table.router_begin(), m_node.m_table.router_end()) << " routers";
#endif
for (routing_table::router_iterator i = m_node.m_table.router_begin()