fixed index of dht stats log and added/fixed dht log parsing and graphin tools

parse_dht_log.py

@@ -1,34 +1,38 @@
 import sys
 import os
 
-up_time_quanta = 60
+up_time_quanta = 2000
 
 f = open(sys.argv[1])
 
 announce_histogram = {}
 node_uptime_histogram = {}
 
-for i in xrange(0, 50): announce_histogram[i] = 0
-for i in xrange(0, 5000, up_time_quanta): node_uptime_histogram[i] = 0
-
 counter = 0;
 
 for line in f:
 	counter += 1
-	if counter % 1000 == 0:
-		print '\r%d' % counter,
+#	if counter % 1000 == 0:
+#		print '\r%d' % counter,
 	try:
 		if 'distance:' in line:
 			l = line.split(' ')
+			idx = l.index('distance:')
 
-			d = int(l[4])
+			d = int(l[idx+1].strip())
+			if not d in announce_histogram: announce_histogram[d] = 0
 			announce_histogram[d] += 1
 		if 'NODE FAILED' in line:
 			l = line.split(' ')
-			if int(l[9]) != 1: continue;
-			d = int(l[11])
-			node_uptime_histogram[d - (d % up_time_quanta)] += 1
-	except:
+			idx = l.index('fails:')
+			if int(l[idx+1].strip()) != 1: continue;
+			idx = l.index('up-time:')
+			d = int(l[idx+1].strip())
+			# quantize
+			d = d - (d % up_time_quanta)
+			if not d in node_uptime_histogram: node_uptime_histogram[d] = 0
+			node_uptime_histogram[d] += 1
+	except Exception, e:
 		print line.split(' ')
 
 out = open('dht_announce_distribution.dat', 'w+')
@@ -67,3 +71,4 @@ out.close()
 
 os.system('gnuplot dht.gnuplot');
 
+

parse_dht_stats.py

@@ -0,0 +1,52 @@
+import sys
+import os
+
+gnuplot_scripts = []
+
+def gen_stats_gnuplot(name, y, lines):
+
+	global gnuplot_scripts
+
+	stat = open(sys.argv[1])
+	line = stat.readline()
+	while not 'minute:' in line:
+		line = stat.readline()
+
+	names = line.strip().split(':')
+	counter = 1
+	for i in names:
+		print '%d: %s' % (counter, i)
+		counter += 1
+
+	out = open('%s.gnuplot' % name, 'w+')
+	out.write('''
+set term png size 1200,700 small
+set output "%s.png"
+set title "%s"
+set ylabel "%s"
+set xlabel "time (minutes)"
+plot ''' % (name, name.strip('_'), y))
+	first = True
+	for i in lines:
+		if not first:
+			out.write(', \\\n')
+		first = False
+		out.write('"%s" using 1:%d title "%s" with lines' % (sys.argv[1], names.index(i)+1, i))
+	out.write('\n')
+
+	out.write('''set terminal postscript
+set output "%s.ps"
+replot
+''' % (name))
+	out.close()
+	gnuplot_scripts += [name]
+
+gen_stats_gnuplot('dht_routing_table_size', 'nodes', ['active nodes','passive nodes'])
+gen_stats_gnuplot('dht_tracker_table_size', '', ['num torrents'])
+gen_stats_gnuplot('dht_announces', 'number per minute', ['announces per min', 'failed announces per min'])
+gen_stats_gnuplot('dht_clients', 'number per minute', ['total msgs per min', 'az msgs per min', 'ut msgs per min', 'lt msgs per min', 'mp msgs per min', 'gr msgs per min'])
+gen_stats_gnuplot('dht_rate', 'bytes per second',  ['bytes in per sec', 'bytes out per sec'])
+gen_stats_gnuplot('dht_errors', '',  ['error replies sent', 'error queries recvd'])
+
+for i in gnuplot_scripts:
+	os.system('gnuplot %s.gnuplot' % i);

src/kademlia/dht_tracker.cpp

@@ -348,20 +348,20 @@ namespace libtorrent { namespace dht
 			first = false;
 			pc << "\n\n *****   starting log at " << time_now_string() << "   *****\n\n"
 				<< "minute:active nodes:passive nodes"
-				":ping replies sent:ping queries recvd:ping"
-				":ping replies sent:ping queries recvd:ping"
+				":ping replies sent:ping queries recvd"
+				":ping replies bytes sent:ping queries bytes recvd"
+				":find_node replies sent:find_node queries recv"
 				":find_node replies bytes sent:find_node queries bytes recv"
-				":find_node replies bytes sent:find_node queries bytes recv"
-				":get_peers replies sent:get_peers queries recvd:get_peers"
+				":get_peers replies sent:get_peers queries recvd"
 				":get_peers replies bytes sent:get_peers queries bytes recv"
-				":announce_peer replies sent:announce_peer queries recvd:announce_peer"
+				":announce_peer replies sent:announce_peer queries recvd"
 				":announce_peer replies bytes sent:announce_peer queries bytes recv"
-				":error replies sent:error queries recvd:error"
+				":error replies sent:error queries recvd"
 				":error replies bytes sent:error queries bytes recv"
 				":num torrents:num peers:announces per min"
 				":failed announces per min:total msgs per min"
-				":ut msgs per min:lt msgs per min:mp msgs per min"
-				":gr msgs per min:bytes in per sec:bytes out per sec"
+				":az msgs per min:ut msgs per min:lt msgs per min:mp msgs per min"
+				":gr msgs per min:mo msgs per min:bytes in per sec:bytes out per sec"
 				":queries out bytes per sec\n\n";
 		}