Merge pull request #379 from arvidn/session_stats-fix

session_stats

simulation/test_swarm.cpp

@@ -35,6 +35,7 @@ POSSIBILITY OF SUCH DAMAGE.
 #include "libtorrent/alert.hpp"
 #include "libtorrent/alert_types.hpp"
 #include "libtorrent/session.hpp"
+#include "libtorrent/session_stats.hpp"
 
 using namespace libtorrent;
 
@@ -78,6 +79,48 @@ TORRENT_TEST(plain)
 		});
 }
 
+TORRENT_TEST(session_stats)
+{
+	std::vector<stats_metric> stats = session_stats_metrics();
+	int const downloading_idx = find_metric_idx("ses.num_downloading_torrents");
+	TEST_CHECK(downloading_idx >= 0);
+	int const evicted_idx = find_metric_idx("ses.torrent_evicted_counter");
+	TEST_CHECK(evicted_idx >= 0);
+	int const incoming_extended_idx = find_metric_idx("ses.num_incoming_extended");
+	TEST_CHECK(incoming_extended_idx >= 0);
+
+	setup_swarm(2, swarm_test::download
+		// add session
+		, [](lt::settings_pack& pack) {}
+		// add torrent
+		, [](lt::add_torrent_params& params) {}
+		// on alert
+		, [=](lt::alert const* a, lt::session& ses)
+		{
+			lt::session_stats_alert const* ss = lt::alert_cast<session_stats_alert>(a);
+			if (!ss) return;
+
+			// there's one downloading torrent
+			TEST_EQUAL(ss->values[downloading_idx], 1);
+			TEST_EQUAL(ss->values[evicted_idx], 0);
+			TEST_EQUAL(ss->values[incoming_extended_idx], 1);
+		}
+		// terminate
+		, [](int ticks, lt::session& ses) -> bool
+		{
+			ses.post_session_stats();
+			if (ticks > 75)
+			{
+				TEST_ERROR("timeout");
+				return true;
+			}
+			if (!is_seed(ses)) return false;
+			printf("completed in %d ticks\n", ticks);
+			return true;
+		});
+}
+
+
 TORRENT_TEST(suggest)
 {
 	setup_swarm(2, swarm_test::download

src/alert.cpp

@@ -1633,9 +1633,20 @@ namespace libtorrent {
 
 	std::string session_stats_alert::message() const
 	{
+		// this specific output is parsed by tools/parse_session_stats.py
+		// if this is changed, that parser should also be changed
 		char msg[100];
-		snprintf(msg, sizeof(msg), "session stats (%d values)", int(sizeof(values)));
-		return msg;
+		snprintf(msg, sizeof(msg), "session stats (%d values): "
+			, int(sizeof(values)/sizeof(values[0])));
+		std::string ret = msg;
+		bool first = true;
+		for (int i = 0; i < sizeof(values)/sizeof(values[0]); ++i)
+		{
+			snprintf(msg, sizeof(msg), first ? "%" PRIu64 : ", %" PRIu64, values[i]);
+			first = false;
+			ret += msg;
+		}
+		return ret;
 	}
 
 	dht_stats_alert::dht_stats_alert(aux::stack_allocator&

src/session_impl.cpp

@@ -93,6 +93,7 @@ const rlim_t rlim_infinity = RLIM_INFINITY;
 #include "libtorrent/peer_connection_handle.hpp"
 #include "libtorrent/ip_filter.hpp"
 #include "libtorrent/socket.hpp"
+#include "libtorrent/session_stats.hpp"
 #include "libtorrent/aux_/session_impl.hpp"
 #ifndef TORRENT_DISABLE_DHT
 #include "libtorrent/kademlia/dht_tracker.hpp"
@@ -553,7 +554,24 @@ namespace aux {
 		TORRENT_ASSERT(is_single_thread());
 
 #ifndef TORRENT_DISABLE_LOGGING
+		if (m_alerts.should_post<log_alert>())
+		{
 			session_log(" *** session thread init");
+
+			// this specific output is parsed by tools/parse_session_stats.py
+			// if this is changed, that parser should also be changed
+			std::string stats_header = "session stats header: ";
+			std::vector<stats_metric> stats = session_stats_metrics();
+			std::sort(stats.begin(), stats.end()
+				, boost::bind(&stats_metric::value_index, _1)
+				< boost::bind(&stats_metric::value_index, _2));
+			for (int i = 0; i < stats.size(); ++i)
+			{
+				if (i > 0) stats_header += ", ";
+				stats_header += stats[i].name;
+			}
+			m_alerts.emplace_alert<log_alert>(stats_header.c_str());
+		}
 #endif
 
 		// this is where we should set up all async operations. This
@@ -4371,7 +4389,6 @@ retry:
 
 		char buf[1024];
 		vsnprintf(buf, sizeof(buf), fmt, v);
-
 		m_alerts.emplace_alert<log_alert>(buf);
 	}
 #endif

src/session_stats.cpp

@@ -163,6 +163,7 @@ namespace libtorrent
 		METRIC(net, on_lsd_peer_counter)
 		METRIC(net, on_udp_counter)
 		METRIC(net, on_accept_counter)
+		METRIC(net, on_disk_queue_counter)
 		METRIC(net, on_disk_counter)
 
 		// total number of bytes sent and received by the session
@@ -215,6 +216,10 @@ namespace libtorrent
 		METRIC(ses, num_queued_download_torrents)
 		METRIC(ses, num_error_torrents)
 
+		// the number of torrents that don't have the
+		// IP filter applied to them.
+		METRIC(ses, non_filter_torrents)
+
 		// the number of torrents that are currently loaded
 		METRIC(ses, num_loaded_torrents)
 		METRIC(ses, num_pinned_torrents)
@@ -467,6 +472,7 @@ namespace libtorrent
 		METRIC(utp, num_utp_connected)
 		METRIC(utp, num_utp_fin_sent)
 		METRIC(utp, num_utp_close_wait)
+		METRIC(utp, num_utp_deleted)
 
 		// the buffer sizes accepted by
 		// socket send and receive calls respectively.

test/test_session.cpp

@@ -31,10 +31,13 @@ POSSIBILITY OF SUCH DAMAGE.
 */
 
 #include "libtorrent/session.hpp"
+#include <boost/bind.hpp>
 
 #include "test.hpp"
 #include "setup_transfer.hpp"
 #include "libtorrent/alert_types.hpp"
+#include "libtorrent/session_stats.hpp"
+#include "libtorrent/performance_counters.hpp"
 
 using namespace libtorrent;
 namespace lt = libtorrent;
@@ -88,3 +91,18 @@ TORRENT_TEST(session)
 	// the session object
 }
 
+TORRENT_TEST(session_stats)
+{
+	std::vector<stats_metric> stats = session_stats_metrics();
+	std::sort(stats.begin(), stats.end()
+		, boost::bind(&stats_metric::value_index, _1)
+		< boost::bind(&stats_metric::value_index, _2));
+
+	TEST_EQUAL(stats.size(), lt::counters::num_counters);
+	// make sure every stat index is represented in the stats_metric vector
+	for (int i = 0; i < int(stats.size()); ++i)
+	{
+		TEST_EQUAL(stats[i].value_index, i);
+	}
+}
+

tools/parse_session_stats.py

@@ -1,21 +1,61 @@
 #!/usr/bin/env python
-# Copyright Arvid Norberg 2008. Use, modification and distribution is
-# subject to the Boost Software License, Version 1.0. (See accompanying
-# file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
+
+# Copyright (c) 2016, Arvid Norberg
+# All rights reserved.
+# 
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+# 
+#     * Redistributions of source code must retain the above copyright
+#       notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above copyright
+#       notice, this list of conditions and the following disclaimer in
+#       the documentation and/or other materials provided with the distribution.
+#     * Neither the name of the author nor the names of its
+#       contributors may be used to endorse or promote products derived
+#       from this software without specific prior written permission.
+# 
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+# POSSIBILITY OF SUCH DAMAGE.
+
+# this script can parse and generate reports from the alert log from a
+# libtorrent session
 
 import os, sys, time, os, math
 from multiprocessing.pool import ThreadPool
 
 thread_pool = ThreadPool(8)
 
+output_dir = 'session_stats_report'
+
 stat = open(sys.argv[1])
 line = stat.readline()
-while not 'second:' in line:
+while not 'session stats header:' in line:
 	line = stat.readline()
 
-keys = line.strip().split(':')[1:]
+keys = line.split('session stats header:')[1].strip().split(', ')
 
-output_dir = 'session_stats_report'
+try: os.mkdir(output_dir)
+except: pass
+data_out = open(os.path.join(output_dir, 'counters.dat'), 'w+')
+
+idx = 0
+for l in stat:
+	if not 'session stats (' in l: continue
+	data_out.write(("%d\t" % idx) + l.split(' values): ')[1].strip().replace(', ', '\t') + '\n')
+	idx += 1
+
+data_out.close()
 
 line_graph = 0
 histogram = 1
@@ -83,11 +123,10 @@ def plot_fun(script):
 	sys.stdout.write('.')
 	sys.stdout.flush()
 
-def gen_report(name, unit, lines, short_unit, generation, log_file, options):
-	try:
-		os.mkdir(output_dir)
-	except: pass
+def to_title(key):
+	return key.replace('_', ' ').replace('.', ' - ')
 
+def gen_report(name, unit, lines, short_unit, generation, log_file, options):
 	filename = os.path.join(output_dir, '%s_%04d.png' % (name, generation))
 	thumb = os.path.join(output_dir, '%s_%04d_thumb.png' % (name, generation))
 
@@ -171,7 +210,7 @@ def gen_report(name, unit, lines, short_unit, generation, log_file, options):
 				graph += '+'
 			axis = 'x1y1'
 			graph += '$%d' % column
-			plot_expression = ' "%s" using 1:(%s) title "%s" axes %s with filledcurves y1=0 lc rgb "%s"' % (log_file, graph, k, axis, colors[color % len(colors)]) + plot_expression
+			plot_expression = ' "%s" using 1:(%s) title "%s" axes %s with filledcurves x1 lc rgb "%s"' % (log_file, graph, to_title(k), axis, colors[color % len(colors)]) + plot_expression
 			first = False
 			color += 1
 		print >>out, plot_expression
@@ -194,7 +233,7 @@ def gen_report(name, unit, lines, short_unit, generation, log_file, options):
 				graph += '-'
 				title += ' - '
 			graph += '$%d' % column
-			title += k
+			title += to_title(k)
 			first = False
 		print >>out, 'plot "%s" using 1:(%s) title "%s" with step' % (log_file, graph, title)
 	else:
@@ -214,7 +253,7 @@ def gen_report(name, unit, lines, short_unit, generation, log_file, options):
 				continue;
 			if not first: print >>out, ', ',
 			axis = 'x1y1'
-			print >>out, ' "%s" using 1:%d title "%s" axes %s with steps lc rgb "%s"' % (log_file, column, k, axis, colors[color % len(colors)]),
+			print >>out, ' "%s" using 1:%d title "%s" axes %s with steps lc rgb "%s"' % (log_file, column, to_title(k), axis, colors[color % len(colors)]),
 			first = False
 			color += 1
 		print >>out, ''
@@ -257,124 +296,293 @@ def gen_html(reports, generations):
 	file.close()
 
 reports = [
-	('torrents', 'num', '', 'number of torrents in different torrent states', ['downloading torrents', 'seeding torrents', \
-		'checking torrents', 'stopped torrents', 'upload-only torrents', 'error torrents', 'queued seed torrents', \
-		'queued download torrents'], {'type':stacked}),
-	('torrents_want_peers', 'num', '', 'number of torrents that want more peers', ['torrents want more peers']),
-	('peers', 'num', '', 'num connected peers', ['peers', 'connecting peers'], {'type':stacked}),
-	('peers_max', 'num', '', 'num connected peers', ['peers', 'connecting peers', 'max connections', 'total peers']),
-	('peer_churn', 'num', '', 'connecting and disconnecting peers', ['connecting peers', 'connection attempts']),
-	('new_peers', 'num', '', '', ['incoming connections', 'connection attempts']),
-	('connection_attempts', 'num', '', '', ['connection attempt loops', 'connection attempts']),
-	('peer_limits', 'num', '', 'number of connections per limit', ['average peers per limit']),
-	('pieces', 'num', '', 'number completed pieces', ['total pieces', 'pieces flushed', 'pieces passed', 'pieces failed']),
-	('connect_candidates', 'num', '', 'number of peers we know of that we can connect to', ['connect candidates']),
-	('peers_list_size', 'num', '', 'number of known peers (not necessarily connected)', ['num list peers']),
-	('overall_rates', 'rate', 'B/s', 'download and upload rates', ['uploaded bytes', 'downloaded bytes', 'upload rate', 'download rate', 'smooth upload rate', 'smooth download rate']),
-	('disk_write_queue', 'Bytes', 'B', 'bytes queued up by peers, to be written to disk', ['disk write queued bytes']),
-	('peers_requests', 'num', '', 'incoming piece request rate', ['piece requests', 'piece rejects', 'max piece requests', 'invalid piece requests', 'choked piece requests', 'cancelled piece requests']),
-	('peers_upload_max', 'num', '', 'number of peers by state wrt. uploading', ['peers up interested', 'peers up unchoked', 'peers up requests', 'peers disk-up', 'peers up send buffer', 'peers bw-up', 'max unchoked']),
-	('peers_upload', 'num', '', 'number of peers by state wrt. uploading', ['peers up interested', 'peers up unchoked', 'peers up requests', 'peers disk-up', 'peers up send buffer', 'peers bw-up']),
-	('peers_download', 'num', '', 'number of peers by state wrt. downloading', ['peers down interesting', 'peers down unchoked', 'peers down requests', 'peers disk-down', 'peers bw-down','num end-game peers']),
-	('peer_errors', 'num', '', 'number of peers by error that disconnected them', ['error peers', 'peer disconnects', 'peers eof', 'peers connection reset', 'connect timeouts', 'uninteresting peers disconnect', 'banned for hash failure', 'no memory peer errors', 'too many peers', 'transport timeout peers', 'connection refused peers', 'connection aborted peers', 'permission denied peers', 'no buffer peers', 'host unreachable peers', 'broken pipe peers', 'address in use peers', 'access denied peers', 'invalid argument peers', 'operation aborted peers']),
-	('peer_errors_incoming', 'num', '', 'number of peers by incoming or outgoing connection', ['error incoming peers', 'error outgoing peers']),
-	('peer_errors_transport', 'num', '', 'number of peers by transport protocol', ['error tcp peers', 'error utp peers']),
-	('peer_errors_encryption', 'num', '', 'number of peers by encryption level', ['error encrypted peers', 'error rc4 peers', 'peer disconnects']),
-	('incoming requests', 'num', '', 'incoming 16kiB block requests', ['pending incoming block requests', 'average pending incoming block requests']),
-	('disk_write_time', 'write time', 's', 'distribution of write jobs timing', ['disk write time'], {'type': histogram, 'binwidth': 0.1, 'numbins': 400}),
-	('disk_read_time', 'read time', 's', 'distribution of read jobs timing', ['disk read time'], {'type': histogram, 'binwidth': 0.1, 'numbins': 400}),
-	('waste', '% of all downloaded bytes', '%%', 'proportion of all downloaded bytes that were wasted', ['% failed payload bytes', '% wasted payload bytes', '% protocol bytes'], {'type':stacked}),
-	('waste by source', 'num wasted bytes', 'B', 'what is causing the waste', [ 'redundant timed-out', 'redundant cancelled', 'redundant unknown', 'redundant seed', 'redundant end-game', 'redundant closing'], {'type':stacked}),
-	('average_disk_time_absolute', 'job time', 's', 'running averages of timings of disk operations', ['disk read time', 'disk write time', 'disk hash time']),
-	('disk_time', '% of total disk job time', '%%', 'proportion of time spent by the disk thread', ['% read time', '% write time', '% hash time'], {'type': stacked}),
-	('disk_cache_hits', 'blocks (16kiB)', '', '', ['disk block read', 'read cache hits'], {'type':stacked}),
-	('disk_cache', 'blocks (16kiB)', '', 'disk cache size and usage', ['disk buffer allocations', 'read disk cache size', 'disk cache size', 'cache size', 'pinned blocks', 'cache trim low watermark']),
-	('disk_readback', '% of written blocks', '%%', 'portion of written blocks that had to be read back for hash verification', ['% read back']),
-	('disk_queue', 'number of queued disk jobs', '', 'queued disk jobs', ['disk queue size', 'disk read queue size', 'allocated jobs', 'allocated read jobs', 'allocated write jobs']),
-	('disk_iops', 'operations/s', '', 'number of disk operations per second', ['read ops/s', 'write ops/s', 'smooth read ops/s', 'smooth write ops/s']),
-	('disk pending reads', 'Bytes', '', 'number of bytes peers are waiting for to be read from the disk', ['pending reading bytes']),
-	('disk fences', 'num', '', 'number of jobs currently blocked by a fence job', ['blocked jobs']),
-	('fence jobs', 'num', '', 'active fence jobs per type', ['move_storage', 'release_files', 'delete_files', 'check_fastresume', 'save_resume_data', 'rename_file', 'stop_torrent', 'file_priority', 'clear_piece'], {'type':stacked}),
-	('disk threads', 'num', '', 'number of disk threads currently writing', ['num writing threads', 'num running threads']),
-	('mixed mode', 'rate', 'B/s', 'rates by transport protocol', ['TCP up rate','TCP down rate','uTP up rate','uTP down rate','TCP up limit','TCP down limit']),
-	('connection_type', 'num', '', 'peers by transport protocol', ['utp peers','tcp peers']),
-	('uTP delay', 'buffering delay', 's', 'network delays measured by uTP', ['uTP peak send delay','uTP peak recv delay', 'uTP avg send delay', 'uTP avg recv delay']),
-	('uTP send delay histogram', 'buffering delay', 's', 'send delays measured by uTP', ['uTP avg send delay'], {'type': histogram, 'binwidth': 0.05, 'numbins': 100}),
-	('uTP recv delay histogram', 'buffering delay', 's', 'receive delays measured by uTP', ['uTP avg recv delay'], {'type': histogram, 'binwidth': 0.05, 'numbins': 100}),
-	('uTP stats', 'num', '', 'number of uTP sockets by state', ['uTP idle', 'uTP syn-sent', 'uTP connected', 'uTP fin-sent', 'uTP close-wait'], {'type': stacked}),
-	('system memory', '', '', 'virtual memory page count', ['active resident pages', 'inactive resident pages', 'pinned resident pages', 'free pages'], {'type': stacked}),
-	('memory paging', '', '', 'vm disk activity', ['pageins', 'pageouts']),
-	('page faults', '', '', '', ['page faults']),
-	('CPU usage', '%', '', '', ['network thread system time', 'network thread user+system time']),
-	('boost.asio messages', 'events/s', '', 'number of messages posted per second', [ \
-		'read_counter', 'write_counter', 'tick_counter', 'lsd_counter', \
-		'lsd_peer_counter', 'udp_counter', 'accept_counter', 'disk_queue_counter', \
-		'disk_counter'], {'type': stacked}),
-	('send_buffer_sizes', 'num', '', '', ['up 8', 'up 16', 'up 32', 'up 64', 'up 128', 'up 256', \
-		'up 512', 'up 1024', 'up 2048', 'up 4096', 'up 8192', 'up 16384', 'up 32768', 'up 65536', \
-		'up 131072', 'up 262144', 'up 524288', 'up 1048576'], {'type': stacked, 'colors':'gradient18'}),
-	('recv_buffer_sizes', 'num', '', '', ['down 8', 'down 16', 'down 32', 'down 64', 'down 128', \
-		'down 256', 'down 512', 'down 1024', 'down 2048', 'down 4096', 'down 8192', 'down 16384', \
-		'down 32768', 'down 65536', 'down 131072', 'down 262144', 'down 524288', 'down 1048576'], {'type': stacked, 'colors':'gradient18'}),
-	('ARC', 'num pieces', '', '', ['arc LRU ghost pieces', 'arc LRU pieces', 'arc LRU volatile pieces', 'arc LFU pieces', 'arc LFU ghost pieces'], {'allow-negative': True}),
-	('torrent churn', 'num torrents', '', '', ['loaded torrents', 'pinned torrents', 'loaded torrent churn']),
-	('pinned torrents', 'num torrents', '', '', ['pinned torrents']),
-	('loaded torrents', 'num torrents', '', '', ['loaded torrents', 'pinned torrents']),
-	('requests', '', '', '', ['outstanding requests']),
-	('request latency', 'us', '', 'latency from receiving requests to sending response', ['request latency']),
+
+	('torrents', 'num', '', 'number of torrents in different torrent states', [ \
+		'ses.num_downloading_torrents', \
+		'ses.num_seeding_torrents', \
+		'ses.num_checking_torrents', \
+		'ses.num_stopped_torrents', \
+		'ses.num_upload_only_torrents', \
+		'ses.num_error_torrents', \
+		'ses.num_queued_seeding_torrents', \
+		'ses.num_queued_download_torrents' \
+	], {'type':stacked}),
+
+	('peers', 'num', '', 'num connected peers', ['peer.num_peers_connected', 'peer.num_peers_half_open'], {'type':stacked}),
+	('peers_max', 'num', '', 'num connected peers', ['peer.num_peers_connected', 'peer.num_peers_half_open']),
+	('peer_churn', 'num', '', 'connecting and disconnecting peers', ['peer.num_peers_half_open', 'peer.connection_attempts']),
+	('new_peers', 'num', '', '', ['peer.incoming_connections', 'peer.connection_attempts']),
+	('connection_attempts', 'num', '', '', ['peer.connection_attempt_loops', 'peer.connection_attempts']),
+	('pieces', 'num', '', 'number completed pieces', ['ses.num_total_pieces_added', 'ses.num_piece_passed', 'ses.num_piece_failed']),
+	('disk_write_queue', 'Bytes', 'B', 'bytes queued up by peers, to be written to disk', ['disk.queued_write_bytes']),
+
+	('peers_requests', 'num', '', 'incoming piece request rate', [ \
+		'peer.piece_requests', \
+		'peer.max_piece_requests', \
+		'peer.invalid_piece_requests', \
+		'peer.choked_piece_requests', \
+		'peer.cancelled_piece_requests' \
+	]),
+
+	('peers_upload', 'num', '', 'number of peers by state wrt. uploading', [ \
+		'peer.num_peers_up_disk', \
+		'peer.num_peers_up_interested', \
+		'peer.num_peers_up_unchoked_all', \
+		'peer.num_peers_up_unchoked_optimistic', \
+		'peer.num_peers_up_unchoked', \
+		'peer.num_peers_up_requests' \
+	]),
+
+	('peers_download', 'num', '', 'number of peers by state wrt. downloading', [ \
+		'peer.num_peers_down_interested', \
+		'peer.num_peers_down_unchoked', \
+		'peer.num_peers_down_requests', \
+		'peer.num_peers_down_disk' \
+	]),
+
+	('peer_errors', 'num', '', 'number of peers by error that disconnected them', [ \
+		'peer.disconnected_peers', \
+		'peer.eof_peers', \
+		'peer.connreset_peers', \
+		'peer.connrefused_peers', \
+		'peer.connaborted_peers', \
+		'peer.perm_peers', \
+		'peer.buffer_peers', \
+		'peer.unreachable_peers', \
+		'peer.broken_pipe_peers', \
+		'peer.addrinuse_peers', \
+		'peer.no_access_peers', \
+		'peer.invalid_arg_peers', \
+		'peer.aborted_peers' \
+	], {'type':stacked}),
+
+	('peer_errors_incoming', 'num', '', 'number of peers by incoming or outgoing connection', [ \
+		'peer.error_incoming_peers', \
+		'peer.error_outgoing_peers' \
+	]),
+
+	('peer_errors_transport', 'num', '', 'number of peers by transport protocol', [ \
+		'peer.error_tcp_peers', \
+		'peer.error_utp_peers'
+	]),
+
+	('peer_errors_encryption', 'num', '', 'number of peers by encryption level', [ \
+		'peer.error_encrypted_peers', \
+		'peer.error_rc4_peers', \
+	]),
+
+	('incoming requests', 'num', '', 'incoming 16kiB block requests', ['ses.num_incoming_request']),
+
+	('waste', 'downloaded bytes', 'B', 'proportion of all downloaded bytes that were wasted', [ \
+		'net.recv_failed_bytes', \
+		'net.recv_redundant_bytes', \
+		'net.recv_ip_overhead_bytes' \
+	], {'type':stacked}),
+
+	('waste by source', 'num wasted bytes', 'B', 'what is causing the waste', [ \
+		'ses.waste_piece_timed_out', \
+		'ses.waste_piece_cancelled', \
+		'ses.waste_piece_unknown', \
+		'ses.waste_piece_seed', \
+		'ses.waste_piece_end_game', \
+		'ses.waste_piece_closing' \
+	], {'type':stacked}),
+
+	('disk_time', '% of total disk job time', '%%', 'proportion of time spent by the disk thread', ['disk.disk_read_time', 'disk.disk_write_time', 'disk.disk_hash_time'], {'type': stacked}),
+	('disk_cache_hits', 'blocks (16kiB)', '', '', ['disk.num_blocks_read', 'disk.num_blocks_cache_hits'], {'type':stacked}),
+	('disk_cache', 'blocks (16kiB)', '', 'disk cache size and usage', ['disk.disk_blocks_in_use', 'disk.read_cache_blocks', 'disk.write_cache_blocks', 'disk.pinned_blocks']),
+	('disk_readback', '% of written blocks', '%%', 'portion of written blocks that had to be read back for hash verification', ['disk.num_read_back']),
+	('disk_queue', 'number of queued disk jobs', '', 'num disk jobs', ['disk.num_write_jobs', 'disk.num_read_jobs', 'disk.num_jobs', 'disk.queued_disk_jobs', 'disk.blocked_disk_jobs']),
+	('disk fences', 'num', '', 'number of jobs currently blocked by a fence job', ['disk.blocked_disk_jobs']),
+#	('fence jobs', 'num', '', 'active fence jobs per type', ['move_storage', 'release_files', 'delete_files', 'check_fastresume', 'save_resume_data', 'rename_file', 'stop_torrent', 'file_priority', 'clear_piece'], {'type':stacked}),
+	('disk threads', 'num', '', 'number of disk threads currently writing', ['disk.num_writing_threads', 'disk.num_running_threads']),
+#	('mixed mode', 'rate', 'B/s', 'rates by transport protocol', ['TCP up rate','TCP down rate','uTP up rate','uTP down rate','TCP up limit','TCP down limit']),
+
+	('connection_type', 'num', '', 'peers by transport protocol', [ \
+		'peer.num_tcp_peers', \
+		'peer.num_socks5_peers', \
+		'peer.num_http_proxy_peers', \
+		'peer.num_utp_peers', \
+		'peer.num_i2p_peers', \
+		'peer.num_ssl_peers', \
+		'peer.num_ssl_socks5_peers', \
+		'peer.num_ssl_http_proxy_peers', \
+		'peer.num_ssl_utp_peers' \
+	]),
+
+#	('uTP delay', 'buffering delay', 's', 'network delays measured by uTP', ['uTP peak send delay','uTP peak recv delay', 'uTP avg send delay', 'uTP avg recv delay']),
+#	('uTP send delay histogram', 'buffering delay', 's', 'send delays measured by uTP', ['uTP avg send delay'], {'type': histogram, 'binwidth': 0.05, 'numbins': 100}),
+#	('uTP recv delay histogram', 'buffering delay', 's', 'receive delays measured by uTP', ['uTP avg recv delay'], {'type': histogram, 'binwidth': 0.05, 'numbins': 100}),
+
+	('uTP stats', 'num', '', 'number of uTP events', [ \
+		'utp.utp_packet_loss', \
+		'utp.utp_timeout', \
+		'utp.utp_packets_in', \
+		'utp.utp_packets_out', \
+		'utp.utp_fast_retransmit', \
+		'utp.utp_packet_resend', \
+		'utp.utp_samples_above_target', \
+		'utp.utp_samples_below_target', \
+		'utp.utp_payload_pkts_in', \
+		'utp.utp_payload_pkts_out', \
+		'utp.utp_invalid_pkts_in', \
+		'utp.utp_redundant_pkts_in' \
+	], {'type': stacked}),
+
+	('boost.asio messages', 'num events', '', 'number of messages posted', [ \
+		'net.on_read_counter', \
+		'net.on_write_counter', \
+		'net.on_tick_counter', \
+		'net.on_lsd_counter', \
+		'net.on_lsd_peer_counter', \
+		'net.on_udp_counter', \
+		'net.on_accept_counter', \
+		'net.on_disk_counter' \
+		], {'type': stacked}),
+
+	('send_buffer_sizes', 'num', '', '', [ \
+		'sock_bufs.socket_send_size3', \
+		'sock_bufs.socket_send_size4', \
+		'sock_bufs.socket_send_size5', \
+		'sock_bufs.socket_send_size6', \
+		'sock_bufs.socket_send_size7', \
+		'sock_bufs.socket_send_size8', \
+		'sock_bufs.socket_send_size9', \
+		'sock_bufs.socket_send_size10', \
+		'sock_bufs.socket_send_size11', \
+		'sock_bufs.socket_send_size12', \
+		'sock_bufs.socket_send_size13', \
+		'sock_bufs.socket_send_size14', \
+		'sock_bufs.socket_send_size15', \
+		'sock_bufs.socket_send_size16', \
+		'sock_bufs.socket_send_size17', \
+		'sock_bufs.socket_send_size18', \
+		'sock_bufs.socket_send_size19', \
+		'sock_bufs.socket_send_size20' \
+	], {'type': stacked, 'colors':'gradient18'}),
+
+	('recv_buffer_sizes', 'num', '', '', [ \
+		'sock_bufs.socket_recv_size3', \
+		'sock_bufs.socket_recv_size4', \
+		'sock_bufs.socket_recv_size5', \
+		'sock_bufs.socket_recv_size6', \
+		'sock_bufs.socket_recv_size7', \
+		'sock_bufs.socket_recv_size8', \
+		'sock_bufs.socket_recv_size9', \
+		'sock_bufs.socket_recv_size10', \
+		'sock_bufs.socket_recv_size11', \
+		'sock_bufs.socket_recv_size12', \
+		'sock_bufs.socket_recv_size13', \
+		'sock_bufs.socket_recv_size14', \
+		'sock_bufs.socket_recv_size15', \
+		'sock_bufs.socket_recv_size16', \
+		'sock_bufs.socket_recv_size17', \
+		'sock_bufs.socket_recv_size18', \
+		'sock_bufs.socket_recv_size19', \
+		'sock_bufs.socket_recv_size20' \
+		], {'type': stacked, 'colors':'gradient18'}),
+
+	('ARC', 'num pieces', '', '', [ \
+		'disk.arc_mru_ghost_size', \
+		'disk.arc_mru_size', \
+		'disk.arc_volatile_size', \
+		'disk.arc_mfu_size', \
+		'disk.arc_mfu_ghost_size' \
+	], {'allow-negative': True}),
+
+	('torrent churn', 'num torrents', '', '', ['ses.num_loaded_torrents', 'ses.num_pinned_torrents', 'ses.torrent_evicted_counter']),
+	('pinned torrents', 'num torrents', '', '', ['ses.num_pinned_torrents']),
+	('loaded torrents', 'num torrents', '', '', ['ses.num_loaded_torrents', 'ses.num_pinned_torrents']),
+	('request latency', 'us', '', 'latency from receiving requests to sending response', ['disk.request_latency']),
 	('incoming messages', 'num', '', 'number of received bittorrent messages, by type', [ \
-		'num_incoming_choke', 'num_incoming_unchoke', 'num_incoming_interested', \
-		'num_incoming_not_interested', 'num_incoming_have', 'num_incoming_bitfield', \
-		'num_incoming_request', 'num_incoming_piece', 'num_incoming_cancel', \
-		'num_incoming_dht_port', 'num_incoming_suggest', 'num_incoming_have_all', \
-		'num_incoming_have_none', 'num_incoming_reject', 'num_incoming_allowed_fast', \
-		'num_incoming_ext_handshake', 'num_incoming_pex', 'num_incoming_metadata', 'num_incoming_extended' \
+		'ses.num_incoming_choke', \
+		'ses.num_incoming_unchoke', \
+		'ses.num_incoming_interested', \
+		'ses.num_incoming_not_interested', \
+		'ses.num_incoming_have', \
+		'ses.num_incoming_bitfield', \
+		'ses.num_incoming_request', \
+		'ses.num_incoming_piece', \
+		'ses.num_incoming_cancel', \
+		'ses.num_incoming_dht_port', \
+		'ses.num_incoming_suggest', \
+		'ses.num_incoming_have_all', \
+		'ses.num_incoming_have_none', \
+		'ses.num_incoming_reject', \
+		'ses.num_incoming_allowed_fast', \
+		'ses.num_incoming_ext_handshake', \
+		'ses.num_incoming_pex', \
+		'ses.num_incoming_metadata', \
+		'ses.num_incoming_extended' \
 	], {'type': stacked}),
 	('outgoing messages', 'num', '', 'number of sent bittorrent messages, by type', [ \
-		'num_outgoing_choke', 'num_outgoing_unchoke', 'num_outgoing_interested', \
-		'num_outgoing_not_interested', 'num_outgoing_have', 'num_outgoing_bitfield', \
-		'num_outgoing_request', 'num_outgoing_piece', 'num_outgoing_cancel', \
-		'num_outgoing_dht_port', 'num_outgoing_suggest', 'num_outgoing_have_all', \
-		'num_outgoing_have_none', 'num_outgoing_reject', 'num_outgoing_allowed_fast', \
-		'num_outgoing_ext_handshake', 'num_outgoing_pex', 'num_outgoing_metadata', 'num_outgoing_extended' \
+		'ses.num_outgoing_choke', \
+		'ses.num_outgoing_unchoke', \
+		'ses.num_outgoing_interested', \
+		'ses.num_outgoing_not_interested', \
+		'ses.num_outgoing_have', \
+		'ses.num_outgoing_bitfield', \
+		'ses.num_outgoing_request', \
+		'ses.num_outgoing_piece', \
+		'ses.num_outgoing_cancel', \
+		'ses.num_outgoing_dht_port', \
+		'ses.num_outgoing_suggest', \
+		'ses.num_outgoing_have_all', \
+		'ses.num_outgoing_have_none', \
+		'ses.num_outgoing_reject', \
+		'ses.num_outgoing_allowed_fast', \
+		'ses.num_outgoing_ext_handshake', \
+		'ses.num_outgoing_pex', \
+		'ses.num_outgoing_metadata', \
+		'ses.num_outgoing_extended' \
 	], {'type': stacked}),
 	('request in balance', 'num', '', 'request and piece message balance', [ \
-		'num_incoming_request', 'num_outgoing_piece', \
-		'num_outgoing_reject'  \
+		'ses.num_incoming_request', \
+		'ses.num_outgoing_piece', \
+		'ses.num_outgoing_reject', \
 	 ], {'type': diff}),
 	('request out balance', 'num', '', 'request and piece message balance', [ \
-		'num_outgoing_request', 'num_incoming_piece', \
-		'num_incoming_reject'  \
+		'ses.num_outgoing_request', \
+		'ses.num_incoming_piece', \
+		'ses.num_incoming_reject', \
 	 ], {'type': diff}),
-#	('absolute_waste', 'num', '', ['failed bytes', 'redundant bytes', 'download rate']),
 
 #somewhat uninteresting stats
-	('tick_rate', 'time between ticks', 's', '', ['tick interval', 'tick residual']),
-	('peer_dl_rates', 'num', '', 'peers split into download rate buckets', ['peers down 0', 'peers down 0-2', 'peers down 2-5', 'peers down 5-10', 'peers down 50-100', 'peers down 100-'], {'type':stacked, 'colors':'gradient6'}),
-	('peer_dl_rates2', 'num', '', 'peers split into download rate buckets (only downloading peers)', ['peers down 0-2', 'peers down 2-5', 'peers down 5-10', 'peers down 50-100', 'peers down 100-'], {'type':stacked, 'colors':'gradient6'}),
-	('peer_ul_rates', 'num', '', 'peers split into upload rate buckets', ['peers up 0', 'peers up 0-2', 'peers up 2-5', 'peers up 5-10', 'peers up 50-100', 'peers up 100-'], {'type':stacked, 'colors':'gradient6'}),
-	('peer_ul_rates2', 'num', '', 'peers split into upload rate buckets (only uploading peers)', ['peers up 0-2', 'peers up 2-5', 'peers up 5-10', 'peers up 50-100', 'peers up 100-'], {'type':stacked, 'colors':'gradient6'}),
-	('piece_picker_invocations', 'invocations of piece picker', '', '', ['reject piece picks', \
-		'unchoke piece picks', 'incoming redundant piece picks', \
-		'incoming piece picks', 'end game piece picks', 'snubbed piece picks', 'interesting piece picks', 'hash fail piece picks'], \
-		{'type':stacked}),
+#	('peer_dl_rates', 'num', '', 'peers split into download rate buckets', ['peers down 0', 'peers down 0-2', 'peers down 2-5', 'peers down 5-10', 'peers down 50-100', 'peers down 100-'], {'type':stacked, 'colors':'gradient6'}),
+#	('peer_dl_rates2', 'num', '', 'peers split into download rate buckets (only downloading peers)', ['peers down 0-2', 'peers down 2-5', 'peers down 5-10', 'peers down 50-100', 'peers down 100-'], {'type':stacked, 'colors':'gradient6'}),
+#	('peer_ul_rates', 'num', '', 'peers split into upload rate buckets', ['peers up 0', 'peers up 0-2', 'peers up 2-5', 'peers up 5-10', 'peers up 50-100', 'peers up 100-'], {'type':stacked, 'colors':'gradient6'}),
+#	('peer_ul_rates2', 'num', '', 'peers split into upload rate buckets (only uploading peers)', ['peers up 0-2', 'peers up 2-5', 'peers up 5-10', 'peers up 50-100', 'peers up 100-'], {'type':stacked, 'colors':'gradient6'}),
+
+	('piece_picker_invocations', 'invocations of piece picker', '', '', [ \
+		'picker.reject_piece_picks', \
+		'picker.unchoke_piece_picks', \
+		'picker.incoming_redundant_piece_picks', \
+		'picker.incoming_piece_picks', \
+		'picker.end_game_piece_picks', \
+		'picker.snubbed_piece_picks', \
+		'picker.interesting_piece_picks', \
+		'picker.hash_fail_piece_picks' \
+	], {'type':stacked}),
 	('piece_picker_loops', 'loops through piece picker', '', '', [ \
-		'piece_picker_partial_loops', 'piece_picker_suggest_loops', 'piece_picker_sequential_loops', 'piece_picker_reverse_rare_loops',
-		'piece_picker_rare_loops', 'piece_picker_rand_start_loops', 'piece_picker_rand_loops', 'piece_picker_busy_loops'], {'type': stacked}),
-	('picker_partials', 'pieces', '', '', ['num downloading partial pieces', 'num full partial pieces', 'num finished partial pieces', \
-		'num 0-priority partial pieces'], {'type':stacked}),
-	('picker_full_partials_distribution', 'full pieces', '', '', ['num full partial pieces'], {'type': histogram, 'binwidth': 5, 'numbins': 120}),
-	('picker_partials_distribution', 'partial pieces', '', '', ['num downloading partial pieces'], {'type': histogram, 'binwidth': 5, 'numbins': 120})
+		'picker.piece_picker_partial_loops', \
+		'picker.piece_picker_suggest_loops', \
+		'picker.piece_picker_sequential_loops', \
+		'picker.piece_picker_reverse_rare_loops', \
+		'picker.piece_picker_rare_loops', \
+		'picker.piece_picker_rand_start_loops', \
+		'picker.piece_picker_rand_loops', \
+		'picker.piece_picker_busy_loops' \
+	], {'type': stacked}),
+
+#	('picker_full_partials_distribution', 'full pieces', '', '', ['num full partial pieces'], {'type': histogram, 'binwidth': 5, 'numbins': 120}),
+#	('picker_partials_distribution', 'partial pieces', '', '', ['num downloading partial pieces'], {'type': histogram, 'binwidth': 5, 'numbins': 120})
 ]
 
 print 'generating graphs'
-log_file_path, log_file = os.path.split(sys.argv[1])
-# count the number of log files (generations)
-log_file_list = log_file.split('.')
-g = int(log_file_list[1])
+g = 0
 generations = []
 scripts = []
-while os.path.exists(os.path.join(log_file_path, log_file)):
+
 print '[%s] %04d\r[' % (' ' * len(reports), g),
 for i in reports:
 	try: options = i[5]
@@ -382,16 +590,14 @@ while os.path.exists(os.path.join(log_file_path, log_file)):
 	if not 'type' in options:
 		options['type'] = line_graph
 
-		script = gen_report(i[0], i[1], i[4], i[2], g, os.path.join(log_file_path, log_file), options)
+	script = gen_report(i[0], i[1], i[4], i[2], g, os.path.join(output_dir, 'counters.dat'), options)
 	if script != None: scripts.append(script)
+
 generations.append(g)
 g += 1
-	log_file_list[1] = '%04d' % g
-	log_file = '.'.join(log_file_list)
 
 # run gnuplot on all scripts, in parallel
 thread_pool.map(plot_fun, scripts)
-	print '' # newline
 scripts = []
 
 print '\ngenerating html'