forked from premiere/premiere-libtorrent
Merge pull request #553 from arvidn/documentation-updates-1.1
fix documentation session_settings -> settings_pack
This commit is contained in:
Arvid Norberg
commit
2b4033d08c
|
|
@ -10,10 +10,6 @@ DOCS_IMAGES = \
|
|||
docs/cwnd_thumb.png \
|
||||
docs/delays.png \
|
||||
docs/delays_thumb.png \
|
||||
docs/disk_access.png \
|
||||
docs/disk_buffer_before_optimization.png \
|
||||
docs/disk_buffer.png \
|
||||
docs/disk_buffer_sample.png \
|
||||
docs/disk_io.png \
|
||||
docs/hacking.html \
|
||||
docs/im_thumb.jpg \
|
||||
|
|
@ -28,8 +24,6 @@ DOCS_IMAGES = \
|
|||
docs/write_disk_buffers.png \
|
||||
docs/write_disk_buffers.diagram \
|
||||
docs/ziptorrent_thumb.gif \
|
||||
docs/disk_access_elevator.png \
|
||||
docs/disk_access_no_elevator.png\
|
||||
docs/ip_id_v4.png \
|
||||
docs/ip_id_v6.png \
|
||||
docs/hash_distribution.png \
|
||||
|
|
|
|||
|
|
@ -385,7 +385,7 @@ namespace
|
|||
allow_threading_guard guard;
|
||||
s.start_upnp();
|
||||
}
|
||||
#endif
|
||||
#endif // TORRENT_NO_DEPRECATE
|
||||
|
||||
#ifndef TORRENT_NO_DEPRECATE
|
||||
boost::shared_ptr<alert>
|
||||
|
|
@ -792,7 +792,7 @@ void bind_session()
|
|||
.def("settings", &session_get_settings)
|
||||
.def("get_settings", &session_get_settings)
|
||||
#endif
|
||||
.def("set_settings", &session_set_settings)
|
||||
.def("apply_settings", &session_set_settings)
|
||||
#ifndef TORRENT_NO_DEPRECATE
|
||||
#ifndef TORRENT_DISABLE_ENCRYPTION
|
||||
.def("set_pe_settings", allow_threads(<::session::set_pe_settings))
|
||||
|
|
|
|||
Binary file not shown.
|
Before Width: | Height: | Size: 3.0 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 5.6 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 5.2 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 5.0 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 4.7 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 5.2 KiB |
|
|
@ -129,7 +129,7 @@ disk caching
|
|||
------------
|
||||
|
||||
All disk I/O in libtorrent is done asynchronously to the network thread, by the
|
||||
disk io thread. When a block is read, the disk io thread reads all subsequent
|
||||
disk io threads. When a block is read, the disk io thread reads all subsequent
|
||||
blocks from that piece into the read cache, assuming that the peer requesting
|
||||
the block will also request more blocks from the same piece. This decreases the
|
||||
number of syscalls for reading data. It also decreases delay from seeking.
|
||||
|
|
@ -155,62 +155,6 @@ The largest contiguous algorithm is the default and flushes the largest contiguo
|
|||
block of buffers, instead of flushing all blocks belonging to the piece which was
|
||||
written to least recently.
|
||||
|
||||
For version 0.15 a lot of work went into optimizing the cache algorithm, trying
|
||||
to increase the cache hit rate and utilization. The graph to the left shows the
|
||||
memory utilization in 0.14. This cache is a straight forward, fairly naive, implementation.
|
||||
Every block read will also read all subsequent blocks in that piece into the cache.
|
||||
Whenever we need more space, the entire oldest piece is evicted from the cache. Caching
|
||||
writes always takes presedence over the read cache. Whenever a piece is fully downloaded,
|
||||
it is flushed to disk.
|
||||
|
||||
.. image:: disk_buffer_before_optimization.png
|
||||
:width: 49%
|
||||
|
||||
.. image:: disk_buffer.png
|
||||
:width: 49%
|
||||
|
||||
The left graph shows the problem of evicting entire pieces at a time, and waiting until
|
||||
an entire piece is downloaded until flushing it. These graphs were generated for a torrent
|
||||
with fairly large pieces. This means that granularity was poor in 0.14, since it only
|
||||
dealt with entire pieces. In 0.15, the granularity problem has been fixed by evicting one
|
||||
block at a time from the read cache. This maximizes the read cache utilization. The write
|
||||
cache is also flushed when a sufficient number of contiguous blocks have been downloaded
|
||||
for a piece, which is not tied to the piece size anymore. This way the cache scales a lot
|
||||
better with piece sizes.
|
||||
|
||||
The graph to the right shows the same download but with the new optimized disk cache
|
||||
algorithm. It clearly shows an increased utilization, which means higher read hit rates
|
||||
or smaller caches with maintained hit rate.
|
||||
|
||||
high performance disk subsystem
|
||||
-------------------------------
|
||||
|
||||
In some circumstances, the disk cache may not suffice to provide maximum performance.
|
||||
One such example is high performance seeding, to a large number of peers, over a fast
|
||||
up-link. In such a case, the amount of RAM may simply not be enough to cache disk
|
||||
reads. When there's not enough RAM to cache disk reads, the disk throughput would
|
||||
typically degrade to perform as poorly as with no cache at all, with the majority
|
||||
of the time spent waiting for the disk head to seek.
|
||||
|
||||
To solve this problem, libtorrent sorts read requests by their physical offset on the
|
||||
disk. They are processed by having the disk read head sweep back and forth over the drive.
|
||||
|
||||
This makes libtorrent very suitable for large scale, high-throughput seeding.
|
||||
|
||||
.. image:: disk_access_no_elevator.png
|
||||
:width: 49%
|
||||
|
||||
.. image:: disk_access_elevator.png
|
||||
:width: 49%
|
||||
|
||||
These plots illustrates the physical disk offset for reads over time. The left plot
|
||||
is of a run where disk operation re-ordering is turned off and the righ is when it's
|
||||
turned on. The right one has a relatively smooth sine wave shape whereas the left
|
||||
one is more random and involves much longer seeks back and forth over the disk.
|
||||
|
||||
True physical disk offset queries are only supported on newer linux kernels, Mac OS X and
|
||||
Windows 2000 and up.
|
||||
|
||||
network buffers
|
||||
---------------
|
||||
|
||||
|
|
|
|||
|
|
@ -119,7 +119,7 @@ libtorrent starts 3 to 5 threads.
|
|||
* The third thread is the SHA-1 hash thread. By default there's only one hash thread,
|
||||
but on multi-core machines downloading at very high rates, libtorrent can be configured
|
||||
to start any number of hashing threads, to take full use of multi core systems.
|
||||
(see ``session_settings::hashing_threads``).
|
||||
(see ``settings_pack::aio_threads``).
|
||||
|
||||
* The fourth and fifth threads are spawned by asio on systems that don't support
|
||||
asynchronous host name resolution, in order to simulate non-blocking ``getaddrinfo()``.
|
||||
|
|
|
|||
|
|
@ -98,13 +98,12 @@ a list of entries.
|
|||
``create_torrent::add_node()`` takes two arguments, one string and one integer,
|
||||
instead of a pair. The string is the address and the integer is the port.
|
||||
|
||||
``session::set_settings()`` not only accepts a ``session_settings`` object, but also
|
||||
a dictionary with keys matching the names of the members of the ``session_settings`` struct.
|
||||
When calling ``set_settings``, the dictionary does not need to have every settings set,
|
||||
keys that are not present, are set to their default value.
|
||||
``session::apply_settings()`` accepts a dictionary with keys matching the names
|
||||
of settings in settings_pack.
|
||||
When calling ``apply_settings``, the dictionary does not need to have every settings set,
|
||||
keys that are not present are not updated.
|
||||
|
||||
For backwards compatibility, ``session::settings()`` still returns a ``session_settings``
|
||||
struct. To get a python dictionary of the settings, call ``session::get_settings``.
|
||||
To get a python dictionary of the settings, call ``session::get_settings``.
|
||||
|
||||
.. _`library reference`: reference.html
|
||||
|
||||
|
|
|
|||
|
|
@ -28,7 +28,7 @@ digraph no_download {
|
|||
node_upload_mode [label="Is the torrent in upload mode?\n(torrent_status::upload_mode)"];
|
||||
node_bwstate [label="What is the peer read_state set to?\n(peer_info::read_state)"];
|
||||
|
||||
node_dl_limit [label="There is a download rate limit in affect on your peers.\nDo you have a download rate limit set?\n(session_settings::download_rate_limit)"];
|
||||
node_dl_limit [label="There is a download rate limit in affect on your peers.\nDo you have a download rate limit set?\n(settings_pack::download_rate_limit)"];
|
||||
|
||||
node_dl_disk [label="Peers are blocked waiting on the disk.\nThis typically means your disk is overloaded"];
|
||||
|
||||
|
|
@ -36,7 +36,7 @@ digraph no_download {
|
|||
|
||||
// end states
|
||||
|
||||
node_end_queued [label="This means the torrent is 'queued'. i.e. it will\nbe started once the torrents in front of it\ncompletes downloading. To know the queue\norder, see torrent_status::queue_position. To\nconfigure the number of simultaneous downloads,\nsee session_settings::active_limit and\nsession_settings::active_downloads."];
|
||||
node_end_queued [label="This means the torrent is 'queued'. i.e. it will\nbe started once the torrents in front of it\ncompletes downloading. To know the queue\norder, see torrent_status::queue_position. To\nconfigure the number of simultaneous downloads,\nsee settings_pack::active_limit and\nsettings_pack::active_downloads."];
|
||||
|
||||
node_end_stopped [label="This means the torrent is\n'stopped'. To start it call\ntorrent_handle::resume()."];
|
||||
|
||||
|
|
|
|||
184
docs/tuning.rst
184
docs/tuning.rst
|
|
@ -12,8 +12,8 @@ libtorrent manual
|
|||
tuning libtorrent
|
||||
=================
|
||||
|
||||
libtorrent expose most constants used in the bittorrent engine for
|
||||
customization through the ``session_settings``. This makes it possible to
|
||||
libtorrent expose most parameters used in the bittorrent engine for
|
||||
customization through the ``settings_pack``. This makes it possible to
|
||||
test and tweak the parameters for certain algorithms to make a client
|
||||
that fits a wide range of needs. From low memory embedded devices to
|
||||
servers seeding thousands of torrents. The default settings in libtorrent
|
||||
|
|
@ -27,7 +27,7 @@ reducing memory footprint
|
|||
=========================
|
||||
|
||||
These are things you can do to reduce the memory footprint of libtorrent. You get
|
||||
some of this by basing your default ``session_settings`` on the ``min_memory_usage()``
|
||||
some of this by basing your default ``settings_pack`` on the ``min_memory_usage()``
|
||||
setting preset function.
|
||||
|
||||
Keep in mind that lowering memory usage will affect performance, always profile
|
||||
|
|
@ -58,9 +58,9 @@ disable disk cache
|
|||
|
||||
The bulk of the memory libtorrent will use is used for the disk cache. To save
|
||||
the absolute most amount of memory, you can disable the cache by setting
|
||||
``session_settings::cache_size`` to 0. You might want to consider using the cache
|
||||
``settings_pack::cache_size`` to 0. You might want to consider using the cache
|
||||
but just disable caching read operations. You do this by settings
|
||||
``session_settings::use_read_cache`` to false. This is the main factor in how much
|
||||
``settings_pack::use_read_cache`` to false. This is the main factor in how much
|
||||
memory will be used by the client. Keep in mind that you will degrade performance
|
||||
by disabling the cache. You should benchmark the disk access in order to make an
|
||||
informed trade-off.
|
||||
|
|
@ -94,8 +94,8 @@ connection, and might be worth considering if you have a very large number of
|
|||
peer connections. This memory will not be visible in your process, this sets
|
||||
the amount of kernel memory is used for your sockets.
|
||||
|
||||
Change this by setting ``session_settings::recv_socket_buffer_size`` and
|
||||
``session_settings::send_socket_buffer_size``.
|
||||
Change this by setting ``settings_pack::recv_socket_buffer_size`` and
|
||||
``settings_pack::send_socket_buffer_size``.
|
||||
|
||||
peer list size
|
||||
--------------
|
||||
|
|
@ -109,7 +109,7 @@ large number of paused torrents (that are popular) it will be even more
|
|||
significant.
|
||||
|
||||
If you're short of memory, you should consider lowering the limit. 500 is probably
|
||||
enough. You can do this by setting ``session_settings::max_peerlist_size`` to
|
||||
enough. You can do this by setting ``settings_pack::max_peerlist_size`` to
|
||||
the max number of peers you want in a torrent's peer list. This limit applies per
|
||||
torrent. For 5 torrents, the total number of peers in peerlists will be 5 times
|
||||
the setting.
|
||||
|
|
@ -117,7 +117,7 @@ the setting.
|
|||
You should also lower the same limit but for paused torrents. It might even make sense
|
||||
to set that even lower, since you only need a few peers to start up while waiting
|
||||
for the tracker and DHT to give you fresh ones. The max peer list size for paused
|
||||
torrents is set by ``session_settings::max_paused_peerlist_size``.
|
||||
torrents is set by ``settings_pack::max_paused_peerlist_size``.
|
||||
|
||||
The drawback of lowering this number is that if you end up in a position where
|
||||
the tracker is down for an extended period of time, your only hope of finding live
|
||||
|
|
@ -132,7 +132,7 @@ The send buffer watermark controls when libtorrent will ask the disk I/O thread
|
|||
to read blocks from disk, and append it to a peer's send buffer.
|
||||
|
||||
When the send buffer has fewer than or equal number of bytes as
|
||||
``session_settings::send_buffer_watermark``, the peer will ask the disk I/O thread
|
||||
``settings_pack::send_buffer_watermark``, the peer will ask the disk I/O thread
|
||||
for more data to send. The trade-off here is between wasting memory by having too
|
||||
much data in the send buffer, and hurting send rate by starving out the socket,
|
||||
waiting for the disk read operation to complete.
|
||||
|
|
@ -156,7 +156,7 @@ the whole piece in one read call, then hashes it.
|
|||
The second option is to optimize for memory usage instead, where a single buffer
|
||||
is allocated, and the piece is read one block at a time, hashing it as each
|
||||
block is read from the file. For low memory environments, this latter approach
|
||||
is recommended. Change this by settings ``session_settings::optimize_hashing_for_speed``
|
||||
is recommended. Change this by settings ``settings_pack::optimize_hashing_for_speed``
|
||||
to false. This will significantly reduce peak memory usage, especially for
|
||||
torrents with very large pieces.
|
||||
|
||||
|
|
@ -200,7 +200,7 @@ processes that might be of higher importance to the end-user, you can introduce
|
|||
between the disc accesses. This is a direct tradeoff between how fast you can check a
|
||||
torrent and how soft you will hit the disk.
|
||||
|
||||
You control this by setting the ``session_settings::file_checks_delay_per_block`` to greater
|
||||
You control this by setting the ``settings_pack::file_checks_delay_per_block`` to greater
|
||||
than zero. This number is the number of milliseconds to sleep between each read of 16 kiB.
|
||||
|
||||
The sleeps are not necessarily in between each 16 kiB block (it might be read in larger chunks),
|
||||
|
|
@ -221,7 +221,7 @@ purpose of this is because of anti-virus software that hooks on file-open and fi
|
|||
scan the file. Anti-virus software that does that will significantly increase the cost of
|
||||
opening and closing files. However, for a high performance seed, the file open/close might
|
||||
be so frequent that it becomes a significant cost. It might therefore be a good idea to allow
|
||||
a large file descriptor cache. Adjust this though ``session_settings::file_pool_size``.
|
||||
a large file descriptor cache. Adjust this though ``settings_pack::file_pool_size``.
|
||||
|
||||
Don't forget to set a high rlimit for file descriptors in your process as well. This limit
|
||||
must be high enough to keep all connections and files open.
|
||||
|
|
@ -230,33 +230,33 @@ disk cache
|
|||
----------
|
||||
|
||||
You typically want to set the cache size to as high as possible. The
|
||||
``session_settings::cache_size`` is specified in 16 kiB blocks. Since you're seeding,
|
||||
the cache would be useless unless you also set ``session_settings::use_read_cache``
|
||||
``settings_pack::cache_size`` is specified in 16 kiB blocks. Since you're seeding,
|
||||
the cache would be useless unless you also set ``settings_pack::use_read_cache``
|
||||
to true.
|
||||
|
||||
In order to increase the possibility of read cache hits, set the
|
||||
``session_settings::cache_expiry`` to a large number. This won't degrade anything as
|
||||
``settings_pack::cache_expiry`` to a large number. This won't degrade anything as
|
||||
long as the client is only seeding, and not downloading any torrents.
|
||||
|
||||
There's a *guided cache* mode. This means the size of the read cache line that's
|
||||
stored in the cache is determined based on the upload rate to the peer that
|
||||
triggered the read operation. The idea being that slow peers don't use up a
|
||||
disproportional amount of space in the cache. This is enabled through
|
||||
``session_settings::guided_read_cache``.
|
||||
``settings_pack::guided_read_cache``.
|
||||
|
||||
In cases where the assumption is that the cache is only used as a read-ahead, and that no
|
||||
other peer will ever request the same block while it's still in the cache, the read
|
||||
cache can be set to be *volatile*. This means that every block that is requested out of
|
||||
the read cache is removed immediately. This saves a significant amount of cache space
|
||||
which can be used as read-ahead for other peers. To enable volatile read cache, set
|
||||
``session_settings::volatile_read_cache`` to true.
|
||||
``settings_pack::volatile_read_cache`` to true.
|
||||
|
||||
SSD as level 2 cache
|
||||
--------------------
|
||||
|
||||
It is possible to introduce a second level of cache, below the RAM disk cache. This is done
|
||||
by setting ``session_settings::mmap_cache`` to a file path pointing to the SSD drive, and
|
||||
increasing the ``session_settings::cache_size`` to the number of 16 kiB blocks would fit
|
||||
by setting ``settings_pack::mmap_cache`` to a file path pointing to the SSD drive, and
|
||||
increasing the ``settings_pack::cache_size`` to the number of 16 kiB blocks would fit
|
||||
on the drive (or less).
|
||||
|
||||
This will allocate disk buffers (for reading and writing) from a memory region that has
|
||||
|
|
@ -280,7 +280,7 @@ throttle TCP to avoid it taking over all bandwidth. This balances the bandwidth
|
|||
between the two protocols. When running on a network where the bandwidth is in such an
|
||||
abundance that it's virtually infinite, this algorithm is no longer necessary, and might
|
||||
even be harmful to throughput. It is adviced to experiment with the
|
||||
``session_setting::mixed_mode_algorithm``, setting it to ``session_settings::prefer_tcp``.
|
||||
``session_setting::mixed_mode_algorithm``, setting it to ``settings_pack::prefer_tcp``.
|
||||
This setting entirely disables the balancing and unthrottles all connections. On a typical
|
||||
home connection, this would mean that none of the benefits of uTP would be preserved
|
||||
(the modem's send buffer would be full at all times) and uTP connections would for the most
|
||||
|
|
@ -299,7 +299,7 @@ enough to not draining the socket's send buffer before the disk operation comple
|
|||
The watermark is bound to a max value, to avoid buffer sizes growing out of control.
|
||||
The default max send buffer size might not be enough to sustain very high upload rates,
|
||||
and you might have to increase it. It's specified in bytes in
|
||||
``session_settings::send_buffer_watermark``.
|
||||
``settings_pack::send_buffer_watermark``.
|
||||
|
||||
peers
|
||||
-----
|
||||
|
|
@ -311,33 +311,33 @@ infinite, ``session::set_upload_rate_limit()``, passing 0 means infinite.
|
|||
When dealing with a large number of peers, it might be a good idea to have slightly
|
||||
stricter timeouts, to get rid of lingering connections as soon as possible.
|
||||
|
||||
There are a couple of relevant settings: ``session_settings::request_timeout``,
|
||||
``session_settings::peer_timeout`` and ``session_settings::inactivity_timeout``.
|
||||
There are a couple of relevant settings: ``settings_pack::request_timeout``,
|
||||
``settings_pack::peer_timeout`` and ``settings_pack::inactivity_timeout``.
|
||||
|
||||
For seeds that are critical for a delivery system, you most likely want to allow
|
||||
multiple connections from the same IP. That way two people from behind the same NAT
|
||||
can use the service simultaneously. This is controlled by
|
||||
``session_settings::allow_multiple_connections_per_ip``.
|
||||
``settings_pack::allow_multiple_connections_per_ip``.
|
||||
|
||||
In order to always unchoke peers, turn off automatic unchoke
|
||||
``session_settings::auto_upload_slots`` and set the number of upload slots to a large
|
||||
``settings_pack::auto_upload_slots`` and set the number of upload slots to a large
|
||||
number via ``session::set_max_uploads()``, or use -1 (which means infinite).
|
||||
|
||||
torrent limits
|
||||
--------------
|
||||
|
||||
To seed thousands of torrents, you need to increase the ``session_settings::active_limit``
|
||||
and ``session_settings::active_seeds``.
|
||||
To seed thousands of torrents, you need to increase the ``settings_pack::active_limit``
|
||||
and ``settings_pack::active_seeds``.
|
||||
|
||||
SHA-1 hashing
|
||||
-------------
|
||||
|
||||
When downloading at very high rates, it is possible to have the CPU be the bottleneck
|
||||
for passing every downloaded byte through SHA-1. In order to enable calculating SHA-1
|
||||
hashes in parallel, on multi-core systems, set ``session_settings::hashing_threads``
|
||||
to the number of threads libtorrent should start to do SHA-1 hashing. This defaults
|
||||
to 1, and only if that thread is close to saturating one core does it make sense to
|
||||
increase the number of threads.
|
||||
When downloading at very high rates, it is possible to have the CPU be the
|
||||
bottleneck for passing every downloaded byte through SHA-1. In order to enable
|
||||
calculating SHA-1 hashes in parallel, on multi-core systems, set
|
||||
``settings_pack::aio_threads`` to the number of threads libtorrent should
|
||||
perform I/O and do SHA-1 hashing in. Only if that thread is close to saturating
|
||||
one core does it make sense to increase the number of threads.
|
||||
|
||||
scalability
|
||||
===========
|
||||
|
|
@ -390,19 +390,7 @@ the disk I/O. The following table is an overview of these files and what they me
|
|||
+--------------------------+--------------------------------------------------------------+
|
||||
| filename | description |
|
||||
+==========================+==============================================================+
|
||||
| ``disk_io_thread.log`` | This is a log of which operation the disk I/O thread is |
|
||||
| | engaged in, with timestamps. This tells you what the thread |
|
||||
| | is spending its time doing. |
|
||||
| | |
|
||||
+--------------------------+--------------------------------------------------------------+
|
||||
| ``disk_buffers.log`` | This log keeps track of what the buffers allocated from the |
|
||||
| | disk buffer pool are used for. There are 5 categories. |
|
||||
| | receive buffer, send buffer, write cache, read cache and |
|
||||
| | temporary hash storage. This is key when optimizing memory |
|
||||
| | usage. |
|
||||
| | |
|
||||
+--------------------------+--------------------------------------------------------------+
|
||||
| ``disk_access.log`` | This is a low level log of read and write operations, with |
|
||||
| ``file_access.log`` | This is a low level log of read and write operations, with |
|
||||
| | timestamps and file offsets. The file offsets are byte |
|
||||
| | offsets in the torrent (not in any particular file, in the |
|
||||
| | case of a multi-file torrent). This can be used as an |
|
||||
|
|
@ -412,106 +400,12 @@ the disk I/O. The following table is an overview of these files and what they me
|
|||
| | |
|
||||
+--------------------------+--------------------------------------------------------------+
|
||||
|
||||
|
||||
disk_io_thread.log
|
||||
''''''''''''''''''
|
||||
|
||||
The structure of this log is simple. For each line, there are two columns, a timestamp and
|
||||
the operation that was started. There is a special operation called ``idle`` which means
|
||||
it looped back to the top and started waiting for new jobs. If there are more jobs to
|
||||
handle immediately, the ``idle`` state is still there, but the timestamp is the same as the
|
||||
next job that is handled.
|
||||
|
||||
Some operations have a 3:rd column with an optional parameter. ``read`` and ``write`` tells
|
||||
you the number of bytes that were requested to be read or written. ``flushing`` tells you
|
||||
the number of bytes that were flushed from the disk cache.
|
||||
|
||||
This is an example excerpt from a log::
|
||||
|
||||
3702 idle
|
||||
3706 check_fastresume
|
||||
3707 idle
|
||||
4708 save_resume_data
|
||||
4708 idle
|
||||
8230 read 16384
|
||||
8255 idle
|
||||
8431 read 16384
|
||||
|
||||
The script to parse this log and generate a graph is called ``parse_disk_log.py``. It takes
|
||||
the log file as the first command line argument, and produces a file: ``disk_io.png``.
|
||||
The time stamp is in milliseconds since start.
|
||||
|
||||
You can pass in a second, optional, argument to specify the window size it will average
|
||||
the time measurements over. The default is 5 seconds. For long test runs, it might be interesting
|
||||
to increase that number. It is specified as a number of seconds.
|
||||
|
||||
.. image:: disk_io.png
|
||||
|
||||
This is an example graph generated by the parse script.
|
||||
|
||||
disk_buffers.log
|
||||
''''''''''''''''
|
||||
|
||||
The disk buffer log tells you where the buffer memory is used. The log format has a time stamp,
|
||||
the name of the buffer usage which use-count changed, colon, and the new number of blocks that are
|
||||
in use for this particular key. For example::
|
||||
|
||||
23671 write cache: 18
|
||||
23671 receive buffer: 3
|
||||
24153 receive buffer: 2
|
||||
24153 write cache: 19
|
||||
24154 receive buffer: 3
|
||||
24198 receive buffer: 2
|
||||
24198 write cache: 20
|
||||
24202 receive buffer: 3
|
||||
24305 send buffer: 0
|
||||
24305 send buffer: 1
|
||||
24909 receive buffer: 2
|
||||
24909 write cache: 21
|
||||
24910 receive buffer: 3
|
||||
|
||||
The time stamp is in milliseconds since start.
|
||||
|
||||
To generate a graph, use ``parse_disk_buffer_log.py``. It takes the log file as the first
|
||||
command line argument. It generates ``disk_buffer.png``.
|
||||
|
||||
.. image:: disk_buffer_sample.png
|
||||
|
||||
This is an example graph generated by the parse script.
|
||||
|
||||
disk_access.log
|
||||
file_access.log
|
||||
'''''''''''''''
|
||||
|
||||
*The disk access log is now binary*
|
||||
|
||||
The disc access log has three fields. The timestamp (milliseconds since start), operation
|
||||
and offset. The offset is the absolute offset within the torrent (not within a file). This
|
||||
log is only useful when you're downloading a single torrent, otherwise the offsets will not
|
||||
be unique.
|
||||
|
||||
In order to easily plot this directly in gnuplot, without parsing it, there are two lines
|
||||
associated with each read or write operation. The first one is the offset where the operation
|
||||
started, and the second one is where the operation ended.
|
||||
|
||||
Example::
|
||||
|
||||
15437 read 301187072
|
||||
15437 read_end 301203456
|
||||
16651 read 213385216
|
||||
16680 read_end 213647360
|
||||
25879 write 249036800
|
||||
25879 write_end 249298944
|
||||
26811 read 325582848
|
||||
26943 read_end 325844992
|
||||
36736 read 367001600
|
||||
36766 read_end 367263744
|
||||
|
||||
The disk access log does not have any good visualization tool yet. There is however a gnuplot
|
||||
file, ``disk_access.gnuplot`` which assumes ``disk_access.log`` is in the current directory.
|
||||
|
||||
.. image:: disk_access.png
|
||||
|
||||
The density of the disk seeks tells you how hard the drive has to work.
|
||||
The disk access log is a binary file that can be parsed and converted to human
|
||||
readable by the script ``tools/parse_access_log.py``. This tool produces a
|
||||
graphical representation of the disk access and requires ``gnuplot``.
|
||||
|
||||
understanding the disk threads
|
||||
==============================
|
||||
|
|
|
|||
|
|
@ -1243,7 +1243,7 @@ int main(int argc, char* argv[])
|
|||
" previous command line options, so be sure to specify this first\n"
|
||||
" -G Add torrents in seed-mode (i.e. assume all pieces\n"
|
||||
" are present and check hashes on-demand)\n"
|
||||
" -E <num-threads> specify how many hashing threads to use\n"
|
||||
" -E <num-threads> specify how many disk I/O threads to use\n"
|
||||
"\n BITTORRENT OPTIONS\n"
|
||||
" -c <limit> sets the max number of connections\n"
|
||||
" -T <limit> sets the max number of connections per torrent\n"
|
||||
|
|
@ -1401,7 +1401,7 @@ int main(int argc, char* argv[])
|
|||
case 'B': settings.set_int(settings_pack::peer_timeout, atoi(arg)); break;
|
||||
case 'n': settings.set_bool(settings_pack::announce_to_all_tiers, true); --i; break;
|
||||
case 'G': seed_mode = true; --i; break;
|
||||
case 'E': settings.set_int(settings_pack::hashing_threads, atoi(arg)); break;
|
||||
case 'E': settings.set_int(settings_pack::aio_threads, atoi(arg)); break;
|
||||
case 'd': settings.set_int(settings_pack::download_rate_limit, atoi(arg) * 1000); break;
|
||||
case 'u': settings.set_int(settings_pack::upload_rate_limit, atoi(arg) * 1000); break;
|
||||
case 'S':
|
||||
|
|
|
|||
|
|
@ -188,7 +188,7 @@ namespace libtorrent
|
|||
// priorities for torrents in share mode, it will make it not work.
|
||||
//
|
||||
// The share mode has one setting, the share ratio target, see
|
||||
// ``session_settings::share_mode_target`` for more info.
|
||||
// ``settings_pack::share_mode_target`` for more info.
|
||||
flag_share_mode = 0x008,
|
||||
|
||||
// determines if the IP filter should apply to this torrent or not. By
|
||||
|
|
|
|||
|
|
@ -54,7 +54,7 @@ POSSIBILITY OF SUCH DAMAGE.
|
|||
// alerts (warnings, messages and errors from libtorrent). If no alerts have
|
||||
// been posted by libtorrent pop_alerts() will return an empty list.
|
||||
//
|
||||
// By default, only errors are reported. session_settings::alert_mask can be
|
||||
// By default, only errors are reported. settings_pack::alert_mask can be
|
||||
// used to specify which kinds of events should be reported. The alert mask is
|
||||
// comprised by bits from the category_t enum.
|
||||
//
|
||||
|
|
|
|||
|
|
@ -865,7 +865,7 @@ namespace libtorrent
|
|||
|
||||
// this is the limit on the number of outstanding requests
|
||||
// we have to this peer. This is initialized to the settings
|
||||
// in the session_settings structure. But it may be lowered
|
||||
// in the settings_pack. But it may be lowered
|
||||
// if the peer is known to require a smaller limit (like BitComet).
|
||||
// or if the extended handshake sets a limit.
|
||||
// web seeds also has a limit on the queue size.
|
||||
|
|
|
|||
|
|
@ -233,7 +233,7 @@ namespace libtorrent
|
|||
|
||||
// the number of seconds until the current front piece request will time
|
||||
// out. This timeout can be adjusted through
|
||||
// ``session_settings::request_timeout``.
|
||||
// ``settings_pack::request_timeout``.
|
||||
// -1 means that there is not outstanding request.
|
||||
int request_timeout;
|
||||
|
||||
|
|
@ -317,7 +317,7 @@ namespace libtorrent
|
|||
|
||||
// the number of bytes this peer has pending in the disk-io thread.
|
||||
// Downloaded and waiting to be written to disk. This is what is capped
|
||||
// by ``session_settings::max_queued_disk_bytes``.
|
||||
// by ``settings_pack::max_queued_disk_bytes``.
|
||||
int pending_disk_bytes;
|
||||
|
||||
// number of outstanding bytes to read
|
||||
|
|
|
|||
|
|
@ -81,7 +81,7 @@ namespace libtorrent
|
|||
// saved when calling save_state().
|
||||
enum save_state_flags_t
|
||||
{
|
||||
// saves settings (i.e. the session_settings)
|
||||
// saves settings (i.e. the settings_pack)
|
||||
save_settings = 0x001,
|
||||
|
||||
// saves dht_settings
|
||||
|
|
@ -844,7 +844,7 @@ namespace libtorrent
|
|||
// aren't working or fail, will automatically be disabled and packets
|
||||
// will flow without using any proxy. If you want to enforce using a
|
||||
// proxy, even when the proxy doesn't work, enable anonymous_mode in
|
||||
// session_settings.
|
||||
// settings_pack.
|
||||
TORRENT_DEPRECATED
|
||||
void set_proxy(proxy_settings const& s);
|
||||
TORRENT_DEPRECATED
|
||||
|
|
@ -957,7 +957,7 @@ namespace libtorrent
|
|||
// The alert queue in the session will not grow indefinitely. Make sure
|
||||
// to pop periodically to not miss notifications. To control the max
|
||||
// number of alerts that's queued by the session, see
|
||||
// ``session_settings::alert_queue_size``.
|
||||
// ``settings_pack::alert_queue_size``.
|
||||
//
|
||||
// Some alerts are considered so important that they are posted even when
|
||||
// the alert queue is full. Some alerts are considered mandatory and cannot
|
||||
|
|
|
|||
|
|
@ -136,7 +136,7 @@ namespace libtorrent
|
|||
//
|
||||
// Since this setting sets a hard upper limit on cache usage, it
|
||||
// cannot be combined with
|
||||
// ``session_settings::contiguous_recv_buffer``, since that feature
|
||||
// ``settings_pack::contiguous_recv_buffer``, since that feature
|
||||
// treats the ``cache_size`` setting as a soft (but still pretty hard)
|
||||
// limit. The result of combining the two is peers being disconnected
|
||||
// after failing to allocate more disk buffers.
|
||||
|
|
@ -1387,6 +1387,9 @@ namespace libtorrent
|
|||
// received by the metadata extension, i.e. magnet links.
|
||||
max_metadata_size,
|
||||
|
||||
#ifndef TORRENT_NO_DEPRECATE
|
||||
// DEPRECTED: use aio_threads instead
|
||||
|
||||
// ``hashing_threads`` is the number of threads to use for piece hash
|
||||
// verification. It defaults to 1. For very high download rates, on
|
||||
// machines with multiple cores, this could be incremented. Setting it
|
||||
|
|
@ -1394,6 +1397,9 @@ namespace libtorrent
|
|||
// any benefit of setting it to the number of cores. If it's set to 0,
|
||||
// hashing is done in the disk thread.
|
||||
hashing_threads,
|
||||
#else
|
||||
deprecated9,
|
||||
#endif
|
||||
|
||||
// the number of blocks to keep outstanding at any given time when
|
||||
// checking torrents. Higher numbers give faster re-checks but uses
|
||||
|
|
|
|||
|
|
@ -598,7 +598,7 @@ namespace libtorrent
|
|||
// Explicitly sets the upload mode of the torrent. In upload mode, the
|
||||
// torrent will not request any pieces. If the torrent is auto managed,
|
||||
// it will automatically be taken out of upload mode periodically (see
|
||||
// ``session_settings::optimistic_disk_retry``). Torrents are
|
||||
// ``settings_pack::optimistic_disk_retry``). Torrents are
|
||||
// automatically put in upload mode whenever they encounter a disk write
|
||||
// error.
|
||||
//
|
||||
|
|
@ -1101,7 +1101,7 @@ namespace libtorrent
|
|||
// ``set_download_limit`` works the same way but for download bandwidth
|
||||
// instead of upload bandwidth. Note that setting a higher limit on a
|
||||
// torrent then the global limit
|
||||
// (``session_settings::upload_rate_limit``) will not override the global
|
||||
// (``settings_pack::upload_rate_limit``) will not override the global
|
||||
// rate limit. The torrent can never upload more than the global rate
|
||||
// limit.
|
||||
//
|
||||
|
|
@ -1170,7 +1170,7 @@ namespace libtorrent
|
|||
// at the same time on this torrent. If you set this to -1, there will be
|
||||
// no limit. This defaults to infinite. The primary setting controlling
|
||||
// this is the global unchoke slots limit, set by unchoke_slots_limit in
|
||||
// session_settings.
|
||||
// settings_pack.
|
||||
//
|
||||
// ``max_uploads()`` returns the current settings.
|
||||
void set_max_uploads(int max_uploads) const;
|
||||
|
|
@ -1182,7 +1182,7 @@ namespace libtorrent
|
|||
// must be at least 2. The default is unlimited number of connections. If
|
||||
// -1 is given to the function, it means unlimited. There is also a
|
||||
// global limit of the number of connections, set by
|
||||
// ``connections_limit`` in session_settings.
|
||||
// ``connections_limit`` in settings_pack.
|
||||
//
|
||||
// ``max_connections()`` returns the current settings.
|
||||
void set_max_connections(int max_connections) const;
|
||||
|
|
|
|||
|
|
@ -282,7 +282,7 @@ namespace libtorrent
|
|||
|
||||
settings_pack sett;
|
||||
sett.set_int(settings_pack::cache_size, 0);
|
||||
sett.set_int(settings_pack::hashing_threads, 2);
|
||||
sett.set_int(settings_pack::aio_threads, 2);
|
||||
|
||||
// TODO: this should probably be optional
|
||||
alert_manager dummy2(0, 0);
|
||||
|
|
|
|||
|
|
@ -97,7 +97,6 @@ namespace libtorrent
|
|||
set.set_int(settings_pack::checking_mem_usage, 2);
|
||||
|
||||
// don't use any extra threads to do SHA-1 hashing
|
||||
set.set_int(settings_pack::hashing_threads, 0);
|
||||
set.set_int(settings_pack::network_threads, 0);
|
||||
set.set_int(settings_pack::aio_threads, 1);
|
||||
|
||||
|
|
@ -274,11 +273,6 @@ namespace libtorrent
|
|||
// connect to us if they want to
|
||||
set.set_int(settings_pack::max_failcount, 1);
|
||||
|
||||
// we're likely to have more than 4 cores on a high
|
||||
// performance machine. One core is needed for the
|
||||
// network thread
|
||||
set.set_int(settings_pack::hashing_threads, 4);
|
||||
|
||||
// the number of threads to use to call async_write_some
|
||||
// and read_some on peer sockets
|
||||
// this doesn't work. See comment in settings_pack.cpp
|
||||
|
|
|
|||
|
|
@ -316,7 +316,7 @@ namespace libtorrent
|
|||
SET(torrent_connect_boost, 10, 0),
|
||||
SET(alert_queue_size, 1000, &session_impl::update_alert_queue_size),
|
||||
SET(max_metadata_size, 3 * 1024 * 10240, 0),
|
||||
SET(hashing_threads, 1, 0),
|
||||
DEPRECATED_SET(hashing_threads, 1, 0),
|
||||
SET(checking_mem_usage, 256, 0),
|
||||
SET(predictive_piece_announce, 0, 0),
|
||||
SET(aio_threads, 4, &session_impl::update_disk_threads),
|
||||
|
|
|
|||
|
|
@ -12,7 +12,7 @@ EXTRA_DIST = Jamfile \
|
|||
parse_dht_log.py \
|
||||
parse_dht_rtt.py \
|
||||
parse_dht_stats.py \
|
||||
parse_disk_access.py \
|
||||
parse_access_log.py \
|
||||
parse_disk_buffer_log.py\
|
||||
parse_disk_log.py \
|
||||
parse_memory_log.py \
|
||||
|
|
|
|||
|
|
@ -1,102 +0,0 @@
|
|||
#!/usr/bin/env python
|
||||
|
||||
import os, sys, time
|
||||
|
||||
lines = open(sys.argv[1], 'rb').readlines()
|
||||
|
||||
# logfile format:
|
||||
# <time(us)> <key>: <value>
|
||||
# example:
|
||||
# 16434 read cache: 17
|
||||
|
||||
keys = ['read', 'write', 'head movement', 'seek per read byte', 'seek per written byte',
|
||||
'read operations per second', 'write operations per second']
|
||||
colors = ['305030', '503030', '3030f0', '10a010', 'a01010', 'd0d040', 'd040d0']
|
||||
style = ['dots', 'points', 'lines', 'lines', 'lines', 'lines', 'lines']
|
||||
axis = ['x1y1', 'x1y1', 'x1y2', 'x1y2', 'x1y2', 'x1y2', 'x1y2']
|
||||
plot = [True, False, False, False, False, True, False]
|
||||
|
||||
out = open('disk_access_log.dat', 'w+')
|
||||
|
||||
time = 1000000
|
||||
|
||||
last_pos = 0
|
||||
last_t = 0
|
||||
cur_movement = 0
|
||||
cur_read = 0
|
||||
cur_write = 0
|
||||
cur_read_ops = 0
|
||||
cur_write_ops = 0
|
||||
|
||||
for l in lines:
|
||||
try:
|
||||
# strip newline
|
||||
l = l[0:-1].split(' ')
|
||||
t = int(l[0])
|
||||
k = l[1]
|
||||
n = int(l[2])
|
||||
except:
|
||||
print l
|
||||
continue
|
||||
|
||||
read = '-'
|
||||
write = '-'
|
||||
movement = '-'
|
||||
amount_read = '-'
|
||||
amount_write = '-'
|
||||
read_ops = '-'
|
||||
write_ops = '-'
|
||||
if k == 'read':
|
||||
read = '%d' % n
|
||||
cur_read_ops += 1
|
||||
if k == 'write':
|
||||
write = '%d' % n
|
||||
cur_write_ops += 1
|
||||
if k == 'read_end': cur_read += n - last_pos
|
||||
if k == 'write_end': cur_write += n - last_pos
|
||||
|
||||
cur_movement += abs(last_pos - n)
|
||||
last_pos = n
|
||||
|
||||
if last_t + time <= t:
|
||||
movement = '%d' % cur_movement
|
||||
if cur_read > 0:
|
||||
amount_read = '%d' % (cur_movement / cur_read)
|
||||
if cur_write > 0:
|
||||
amount_write = '%d' % (cur_movement / cur_write)
|
||||
read_ops = '%d' % cur_read_ops
|
||||
write_ops = '%d' % cur_write_ops
|
||||
cur_movement = 0
|
||||
cur_read = 0
|
||||
cur_write = 0
|
||||
last_t = t
|
||||
cur_read_ops = 0
|
||||
cur_write_ops = 0
|
||||
|
||||
print >>out, '%d\t%s\t%s\t%s\t%s\t%s\t%s\t%s' % (t, read, write, movement, amount_read, amount_write, read_ops, write_ops)
|
||||
|
||||
out.close()
|
||||
|
||||
out = open('disk_access.gnuplot', 'wb')
|
||||
print >>out, "set term png size 1200,700"
|
||||
print >>out, 'set output "disk_access.png"'
|
||||
print >>out, 'set xrange [*:*]'
|
||||
#print >>out, 'set y2range [0:*]'
|
||||
print >>out, 'set xlabel "time (us)"'
|
||||
print >>out, 'set ylabel "drive offset"'
|
||||
#print >>out, 'set y2label "bytes / %d second(s)"' % (time / 1000)
|
||||
print >>out, "set key box"
|
||||
print >>out, "set tics nomirror"
|
||||
print >>out, "set y2tics auto"
|
||||
print >>out, 'plot',
|
||||
count = 1
|
||||
for k in keys:
|
||||
count += 1
|
||||
if not plot[count-2]: continue
|
||||
print >>out, ' "disk_access_log.dat" using 1:%d title "%s" with %s lt rgb "#%s" axis %s,' \
|
||||
% (count, k, style[count-2], colors[count-2], axis[count-2]),
|
||||
print >>out, 'x=0'
|
||||
out.close()
|
||||
|
||||
os.system('gnuplot disk_access.gnuplot')
|
||||
|
||||
Loading…
Reference in New Issue