wornvirtue
/
premiere-libtorrent
forked from premiere/premiere-libtorrent
1
0
Fork 0
Code Pull Requests Activity

remove stats logging functionality from libtorrent. This has been moved to a helper class part of libtorrent-webui, utility library

This commit is contained in:
Arvid Norberg 2014-10-21 19:58:12 +00:00
parent 6aa0ae9f2e
commit cea5585993
17 changed files with 586 additions and 1415 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
.regression.yml
View File

@ -4,7 +4,7 @@ test_dirs:
features: features:
- variant=release asserts=production debug-symbols=on - variant=release asserts=production debug-symbols=on
- encryption=openssl statistics=on logging=verbose disk-stats=on dht=logging request-log=on allocator=debug debug-iterators=on - encryption=openssl logging=verbose disk-stats=on dht=logging request-log=on allocator=debug debug-iterators=on
# all build_features are built in these directories # all build_features are built in these directories
build_dirs: build_dirs:
@ -17,7 +17,6 @@ build_features:
- deprecated-functions=off - deprecated-functions=off
- exception-handling=off - exception-handling=off
- logging=verbose - logging=verbose
- statistics=on
- asserts=off - asserts=off
- asserts=production debug - asserts=production debug
- asserts=on release - asserts=on release

3
Jamfile
View File

@ -420,9 +420,6 @@ feature.compose <character-set>unicode : <define>_UNICODE <define>UNICODE ;
feature deprecated-functions : on off : composite propagated link-incompatible ; feature deprecated-functions : on off : composite propagated link-incompatible ;
feature.compose <deprecated-functions>off : <define>TORRENT_NO_DEPRECATE ; feature.compose <deprecated-functions>off : <define>TORRENT_NO_DEPRECATE ;
feature statistics : off on : composite propagated link-incompatible ;
feature.compose <statistics>on : <define>TORRENT_STATS ;
feature upnp-logging : off on : composite propagated link-incompatible ; feature upnp-logging : off on : composite propagated link-incompatible ;
feature.compose <upnp-logging>on : <define>TORRENT_UPNP_LOGGING ; feature.compose <upnp-logging>on : <define>TORRENT_UPNP_LOGGING ;

4
build_dist.sh
View File

@ -2,6 +2,10 @@
make distclean make distclean
cd docs
make
cd ..
#clear out any extended attributes that Finder may add #clear out any extended attributes that Finder may add
sudo xattr -r -d com.apple.FinderInfo * sudo xattr -r -d com.apple.FinderInfo *

23
configure.ac
View File

@ -263,15 +263,6 @@ AC_ARG_ENABLE(
[[ARG_ENABLE_DEPRECATED=yes]] [[ARG_ENABLE_DEPRECATED=yes]]
) )
AC_ARG_ENABLE(
[statistics],
[AS_HELP_STRING(
[--enable-statistics],
[enable statistics logging feature [default=no]])],
[[ARG_ENABLE_STATS=$enableval]],
[[ARG_ENABLE_STATS=no]]
)
AC_ARG_ENABLE( AC_ARG_ENABLE(
[disk-stats], [disk-stats],
[AS_HELP_STRING( [AS_HELP_STRING(
@ -402,19 +393,6 @@ AS_CASE(["$ARG_ENABLE_LOGGING"],
AC_MSG_ERROR([Unknown option "$ARG_ENABLE_LOGGING". Use either "yes", "no", "verbose" or "errors".])] AC_MSG_ERROR([Unknown option "$ARG_ENABLE_LOGGING". Use either "yes", "no", "verbose" or "errors".])]
) )
AC_MSG_CHECKING([whether statistics logging should be enabled])
AS_CASE(["$ARG_ENABLE_STATS"],
["yes"|"on"], [
AC_MSG_RESULT([yes])
AC_DEFINE([TORRENT_STATS],[1],[Define to generate a log with statistics.])
],
["no"|"off"], [
AC_MSG_RESULT([no])
],
[AC_MSG_RESULT([$ARG_ENABLE_STATS])
AC_MSG_ERROR([Unknown option "$ARG_ENABLE_STATS". Use either "yes" or "no".])]
)
AC_MSG_CHECKING([whether disk activity logging should be enabled]) AC_MSG_CHECKING([whether disk activity logging should be enabled])
AS_CASE(["$ARG_ENABLE_DISK_STATS"], AS_CASE(["$ARG_ENABLE_DISK_STATS"],
["yes"|"on"], [ ["yes"|"on"], [
@ -659,7 +637,6 @@ Build options:
debug build: ${ARG_ENABLE_DEBUG:-no} debug build: ${ARG_ENABLE_DEBUG:-no}
invariant checks: ${ARG_ENABLE_INVARIANT:-no} invariant checks: ${ARG_ENABLE_INVARIANT:-no}
logging support: ${ARG_ENABLE_LOGGING:-no} logging support: ${ARG_ENABLE_LOGGING:-no}
statistics: ${ARG_ENABLE_STATS:-no}
disk statistics: ${ARG_ENABLE_DISK_STATS:-no} disk statistics: ${ARG_ENABLE_DISK_STATS:-no}
Features: Features:

6
docs/building.rst
View File

@ -577,12 +577,6 @@ defines you can use to control the build.
| | which later can parsed and graphed using | | | which later can parsed and graphed using |
| | ``parse_disk_log.py``. | | | ``parse_disk_log.py``. |
+----------------------------------------+-------------------------------------------------+ +----------------------------------------+-------------------------------------------------+
| ``TORRENT_STATS`` | This will generate a log with transfer rates, |
| | downloading torrents, seeding torrents, peers, |
| | connecting peers and disk buffers in use. The |
| | log can be parsed and graphed with |
| | ``parse_session_stats.py``. |
+----------------------------------------+-------------------------------------------------+
| ``UNICODE`` | If building on windows this will make sure the | | ``UNICODE`` | If building on windows this will make sure the |
| | UTF-8 strings in pathnames are converted into | | | UTF-8 strings in pathnames are converted into |
| | UTF-16 before they are passed to the file | | | UTF-16 before they are passed to the file |

2
docs/reference-Session.html
View File

@ -3,7 +3,7 @@
<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en"> <html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
<head> <head>
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" /> <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
<meta name="generator" content="Docutils 0.11: http://docutils.sourceforge.net/" /> <meta name="generator" content="Docutils 0.12: http://docutils.sourceforge.net/" />
<title>Session</title> <title>Session</title>
<meta name="author" content="Arvid Norberg, arvid&#64;libtorrent.org" /> <meta name="author" content="Arvid Norberg, arvid&#64;libtorrent.org" />
<link rel="stylesheet" type="text/css" href="rst.css" /> <link rel="stylesheet" type="text/css" href="rst.css" />

746
docs/todo.html
View File

File diff suppressed because one or more lines are too long

99
docs/tuning.rst
View File

@ -516,6 +516,8 @@ This is an example graph generated by the parse script.
disk_access.log disk_access.log
''''''''''''''' '''''''''''''''
*The disk access log is now binary*
The disc access log has three fields. The timestamp (milliseconds since start), operation The disc access log has three fields. The timestamp (milliseconds since start), operation
and offset. The offset is the absolute offset within the torrent (not within a file). This and offset. The offset is the absolute offset within the torrent (not within a file). This
log is only useful when you're downloading a single torrent, otherwise the offsets will not log is only useful when you're downloading a single torrent, otherwise the offsets will not
@ -545,74 +547,57 @@ file, ``disk_access.gnuplot`` which assumes ``disk_access.log`` is in the curren
The density of the disk seeks tells you how hard the drive has to work. The density of the disk seeks tells you how hard the drive has to work.
session stats understanding the disk threads
------------- ==============================
By defining ``TORRENT_STATS`` libtorrent will write a log file called ``session_stats/<pid>.<sequence>.log`` which *This section is somewhat outdated, there are potentially more than one disk
is in a format ready to be passed directly into gnuplot. The parser script ``parse_session_stats.py`` thread*
generates a report in ``session_stats_report/index.html``.
The first line in the log contains all the field names, separated by colon:: All disk operations are funneled through a separate thread, referred to as the
disk thread. The main interface to the disk thread is a queue where disk jobs
second:upload rate:download rate:downloading torrents:seeding torrents:peers... are posted, and the results of these jobs are then posted back on the main
thread's io_service.
The rest of the log is one line per second with all the fields' values.
These are the fields:
===================== ===============================================================
field name description
===================== ===============================================================
second the time, in seconds, for this log line
upload rate the number of bytes uploaded in the last second
download rate the number of bytes downloaded in the last second
downloading torrents the number of torrents that are not seeds
seeding torrents the number of torrents that are seed
peers the total number of connected peers
connecting peers the total number of peers attempting to connect (half-open)
disk block buffers the total number of disk buffer blocks that are in use
unchoked peers the total number of unchoked peers
num list peers the total number of known peers, but not necessarily connected
peer allocations the total number of allocations for the peer list pool
peer storage bytes the total number of bytes allocated for the peer list pool
===================== ===============================================================
This is an example of a graph that can be generated from this log:
.. image:: session_stats_peers.png
It shows statistics about the number of peers and peers states. How at the startup
there are a lot of half-open connections, which tapers off as the total number of
peers approaches the limit (50). It also shows how the total peer list slowly but steadily
grows over time. This list is plotted against the right axis, as it has a different scale
as the other fields.
understanding the disk thread
=============================
All disk operations are funneled through a separate thread, referred to as the disk thread.
The main interface to the disk thread is a queue where disk jobs are posted, and the results
of these jobs are then posted back on the main thread's io_service.
A disk job is essentially one of: A disk job is essentially one of:
1. write this block to disk, i.e. a write job. For the most part this is just a matter of sticking the block in the disk cache, but if we've run out of cache space or completed a whole piece, we'll also flush blocks to disk. This is typically very fast, since the OS just sticks these buffers in its write cache which will be flushed at a later time, presumably when the drive head will pass the place on the platter where the blocks go. 1. write this block to disk, i.e. a write job. For the most part this is just a
matter of sticking the block in the disk cache, but if we've run out of
cache space or completed a whole piece, we'll also flush blocks to disk.
This is typically very fast, since the OS just sticks these buffers in its
write cache which will be flushed at a later time, presumably when the drive
head will pass the place on the platter where the blocks go.
2. read this block from disk. The first thing that happens is we look in the cache to see if the block is already in RAM. If it is, we'll return immediately with this block. If it's a cache miss, we'll have to hit the disk. Here we decide to defer this job. We find the physical offset on the drive for this block and insert the job in an ordered queue, sorted by the physical location. At a later time, once we don't have any more non-read jobs left in the queue, we pick one read job out of the ordered queue and service it. The order we pick jobs out of the queue is according to an elevator cursor moving up and down along the ordered queue of read jobs. If we have enough space in the cache we'll read read_cache_line_size number of blocks and stick those in the cache. This defaults to 32 blocks. If the system supports asynchronous I/O (Windows, Linux, Mac OS X, BSD, Solars for instance), jobs will be issued immediately to the OS. This especially increases read throughput, since the OS has a much greater flexibility to reorder the read jobs. 2. read this block from disk. The first thing that happens is we look in the
cache to see if the block is already in RAM. If it is, we'll return
Other disk job consist of operations that needs to be synchronized with the disk I/O, like renaming files, closing files, flushing the cache, updating the settings etc. These are relatively rare though. immediately with this block. If it's a cache miss, we'll have to hit the
disk. Here we decide to defer this job. We find the physical offset on the
drive for this block and insert the job in an ordered queue, sorted by the
physical location. At a later time, once we don't have any more non-read
jobs left in the queue, we pick one read job out of the ordered queue and
service it. The order we pick jobs out of the queue is according to an
elevator cursor moving up and down along the ordered queue of read jobs. If
we have enough space in the cache we'll read read_cache_line_size number of
blocks and stick those in the cache. This defaults to 32 blocks. If the
system supports asynchronous I/O (Windows, Linux, Mac OS X, BSD, Solars for
instance), jobs will be issued immediately to the OS. This especially
increases read throughput, since the OS has a much greater flexibility to
reorder the read jobs.
Other disk job consist of operations that needs to be synchronized with the
disk I/O, like renaming files, closing files, flushing the cache, updating the
settings etc. These are relatively rare though.
contributions contributions
============= =============
If you have added instrumentation for some part of libtorrent that is not covered here, or If you have added instrumentation for some part of libtorrent that is not
if you have improved any of the parser scrips, please consider contributing it back to the covered here, or if you have improved any of the parser scrips, please consider
project. contributing it back to the project.
If you have run tests and found that some algorithm or default value in libtorrent is If you have run tests and found that some algorithm or default value in
suboptimal, please contribute that knowledge back as well, to allow us to improve the library. libtorrent is suboptimal, please contribute that knowledge back as well, to
allow us to improve the library.
If you have additional suggestions on how to tune libtorrent for any specific use case, If you have additional suggestions on how to tune libtorrent for any specific
please let us know and we'll update this document. use case, please let us know and we'll update this document.

50
include/libtorrent/aux_/session_impl.hpp
View File

@ -105,13 +105,6 @@ POSSIBILITY OF SUCH DAMAGE.
#include <boost/asio/ssl/context.hpp> #include <boost/asio/ssl/context.hpp>
#endif #endif
#if defined TORRENT_STATS && defined __MACH__
#include <mach/vm_statistics.h>
#include <mach/mach_init.h>
#include <mach/host_info.h>
#include <mach/mach_host.h>
#endif
namespace libtorrent namespace libtorrent
{ {
@ -161,25 +154,6 @@ namespace libtorrent
struct session_impl; struct session_impl;
struct session_settings; struct session_settings;
#if defined TORRENT_STATS && !defined __MACH__
struct vm_statistics_data_t
{
boost::uint64_t active_count;
boost::uint64_t inactive_count;
boost::uint64_t wire_count;
boost::uint64_t free_count;
boost::uint64_t pageins;
boost::uint64_t pageouts;
boost::uint64_t faults;
};
#endif
struct thread_cpu_usage
{
time_duration user_time;
time_duration system_time;
};
#if defined TORRENT_VERBOSE_LOGGING || defined TORRENT_LOGGING || defined TORRENT_ERROR_LOGGING #if defined TORRENT_VERBOSE_LOGGING || defined TORRENT_LOGGING || defined TORRENT_ERROR_LOGGING
struct tracker_logger; struct tracker_logger;
#endif #endif
@ -1128,30 +1102,6 @@ namespace libtorrent
void received_buffer(int size); void received_buffer(int size);
void sent_buffer(int size); void sent_buffer(int size);
#ifdef TORRENT_STATS
void rotate_stats_log();
void print_log_line(int tick_interval_ms, ptime now);
void enable_stats_logging(bool s);
bool m_stats_logging_enabled;
// the last time we rotated the log file
ptime m_last_log_rotation;
// logger used to write bandwidth usage statistics
FILE* m_stats_logger;
// sequence number for log file. Log files are
// rotated every hour and the sequence number is
// incremented by one
int m_log_seq;
counters m_last_stats_counters;
cache_status m_last_cache_status;
vm_statistics_data_t m_last_vm_stat;
thread_cpu_usage m_network_thread_cpu_usage;
#endif
// each second tick the timer takes a little // each second tick the timer takes a little
// bit longer than one second to trigger. The // bit longer than one second to trigger. The
// extra time it took is accumulated into this // extra time it took is accumulated into this

9
include/libtorrent/disk_buffer_pool.hpp
View File

@ -82,11 +82,6 @@ namespace libtorrent
int block_size() const { return m_block_size; } int block_size() const { return m_block_size; }
#ifdef TORRENT_STATS
int disk_allocations() const
{ return m_allocations; }
#endif
void release_memory(); void release_memory();
boost::uint32_t in_use() const boost::uint32_t in_use() const
@ -192,10 +187,6 @@ namespace libtorrent
boost::pool<page_aligned_allocator> m_pool; boost::pool<page_aligned_allocator> m_pool;
#endif #endif
#if defined TORRENT_STATS
int m_allocations;
#endif
// this is specifically exempt from release_asserts // this is specifically exempt from release_asserts
// since it's a quite costly check. Only for debug // since it's a quite costly check. Only for debug
// builds. // builds.

5
include/libtorrent/session.hpp
View File

@ -998,11 +998,6 @@ namespace libtorrent
void apply_settings(settings_pack const& s); void apply_settings(settings_pack const& s);
aux::session_settings get_settings() const; aux::session_settings get_settings() const;
#ifdef TORRENT_STATS
// internal
void enable_stats_logging(bool s);
#endif
#ifndef TORRENT_NO_DEPRECATE #ifndef TORRENT_NO_DEPRECATE
// ``set_i2p_proxy`` sets the i2p_ proxy, and tries to open a persistant // ``set_i2p_proxy`` sets the i2p_ proxy, and tries to open a persistant
// connection to it. The only used fields in the proxy settings structs // connection to it. The only used fields in the proxy settings structs

11
src/disk_buffer_pool.cpp
View File

@ -121,9 +121,6 @@ namespace libtorrent
, m_pool(block_size, 32) , m_pool(block_size, 32)
#endif #endif
{ {
#if defined TORRENT_STATS
m_allocations = 0;
#endif
#if TORRENT_USE_ASSERTS #if TORRENT_USE_ASSERTS
m_magic = 0x1337; m_magic = 0x1337;
m_settings_set = false; m_settings_set = false;
@ -381,10 +378,6 @@ namespace libtorrent
} }
#endif // TORRENT_USE_MLOCK #endif // TORRENT_USE_MLOCK
#if defined TORRENT_STATS
++m_allocations;
#endif
TORRENT_ASSERT(is_disk_buffer(ret, l)); TORRENT_ASSERT(is_disk_buffer(ret, l));
return ret; return ret;
} }
@ -596,10 +589,6 @@ namespace libtorrent
} }
#endif #endif
#if defined TORRENT_STATS
--m_allocations;
#endif
#if TORRENT_HAVE_MMAP #if TORRENT_HAVE_MMAP
if (m_cache_pool) if (m_cache_pool)
{ {

3
src/lt_trackers.cpp
View File

@ -59,9 +59,6 @@ POSSIBILITY OF SUCH DAMAGE.
#include "libtorrent/io.hpp" #include "libtorrent/io.hpp"
#include "libtorrent/escape_string.hpp" #include "libtorrent/escape_string.hpp"
#include "libtorrent/parse_url.hpp" #include "libtorrent/parse_url.hpp"
#ifdef TORRENT_STATS
#include "libtorrent/aux_/session_impl.hpp"
#endif
namespace libtorrent { namespace namespace libtorrent { namespace
{ {

9
src/session.cpp
View File

@ -1045,16 +1045,7 @@ namespace libtorrent
ret.port = sett.get_int(settings_pack::i2p_port); ret.port = sett.get_int(settings_pack::i2p_port);
return ret; return ret;
} }
#endif // TORRENT_NO_DEPRECATE
#ifdef TORRENT_STATS
void session::enable_stats_logging(bool s)
{
TORRENT_ASYNC_CALL1(enable_stats_logging, s);
}
#endif
#ifndef TORRENT_NO_DEPRECATE
void session::set_max_half_open_connections(int limit) {} void session::set_max_half_open_connections(int limit) {}
int session::max_half_open_connections() const { return 8; } int session::max_half_open_connections() const { return 8; }

994
src/session_impl.cpp
View File

File diff suppressed because it is too large Load Diff

30
src/session_stats.cpp
View File

@ -69,6 +69,18 @@ namespace libtorrent
METRIC(peer, invalid_arg_peers) METRIC(peer, invalid_arg_peers)
METRIC(peer, aborted_peers) METRIC(peer, aborted_peers)
// the total number of incoming piece requests we've received followed
// by the number of rejected piece requests for various reasons.
// max_piece_requests mean we already had too many outstanding requests
// from this peer, so we rejected it. cancelled_piece_requests are ones
// where the other end explicitly asked for the piece to be rejected.
METRIC(peer, piece_requests)
METRIC(peer, max_piece_requests)
METRIC(peer, invalid_piece_requests)
METRIC(peer, choked_piece_requests)
METRIC(peer, cancelled_piece_requests)
METRIC(peer, piece_rejects)
// these counters break down the peer errors into // these counters break down the peer errors into
// whether they happen on incoming or outgoing peers. // whether they happen on incoming or outgoing peers.
METRIC(peer, error_incoming_peers) METRIC(peer, error_incoming_peers)
@ -268,6 +280,15 @@ namespace libtorrent
METRIC(ses, num_outgoing_metadata) METRIC(ses, num_outgoing_metadata)
METRIC(ses, num_outgoing_extended) METRIC(ses, num_outgoing_extended)
// the number of wasted downloaded bytes by reason of the bytes being
// wasted.
METRIC(ses, waste_piece_timed_out)
METRIC(ses, waste_piece_cancelled)
METRIC(ses, waste_piece_unknown)
METRIC(ses, waste_piece_seed)
METRIC(ses, waste_piece_end_game)
METRIC(ses, waste_piece_closing)
// the number of pieces considered while picking pieces // the number of pieces considered while picking pieces
METRIC(picker, piece_picker_partial_loops) METRIC(picker, piece_picker_partial_loops)
METRIC(picker, piece_picker_suggest_loops) METRIC(picker, piece_picker_suggest_loops)
@ -347,15 +368,6 @@ namespace libtorrent
METRIC(disk, disk_hash_time) METRIC(disk, disk_hash_time)
METRIC(disk, disk_job_time) METRIC(disk, disk_job_time)
// the number of wasted downloaded bytes by reason of the bytes being
// wasted.
METRIC(ses, waste_piece_timed_out)
METRIC(ses, waste_piece_cancelled)
METRIC(ses, waste_piece_unknown)
METRIC(ses, waste_piece_seed)
METRIC(ses, waste_piece_end_game)
METRIC(ses, waste_piece_closing)
// The number of nodes in the DHT routing table // The number of nodes in the DHT routing table
METRIC(dht, dht_nodes) METRIC(dht, dht_nodes)

4
src/ut_metadata.cpp
View File

@ -60,10 +60,6 @@ POSSIBILITY OF SUCH DAMAGE.
#include "libtorrent/io.hpp" #include "libtorrent/io.hpp"
#include "libtorrent/performance_counters.hpp" // for counters #include "libtorrent/performance_counters.hpp" // for counters
#ifdef TORRENT_STATS
#include "libtorrent/aux_/session_impl.hpp"
#endif
namespace libtorrent { namespace namespace libtorrent { namespace
{ {
enum enum