update tuning documentation (#1717)

update tuning documentation and make the stats header be posted when logging is disabled (if stats_notifications are enabled)

bindings/python/test.py

@@ -7,6 +7,7 @@ import time
 import os
 import shutil
 import binascii
+import inspect
 
 class test_create_torrent(unittest.TestCase):
 
@@ -238,7 +239,18 @@ class test_session(unittest.TestCase):
 	def test_post_session_stats(self):
 		s = lt.session({'alert_mask': lt.alert.category_t.stats_notification, 'enable_dht': False})
 		s.post_session_stats()
-		a = s.wait_for_alert(1000)
+		alerts = []
+		# first the stats headers log line. but not if logging is disabled
+		if 'log_alert' in [i[0] for i in inspect.getmembers(lt)]:
+			s.wait_for_alert(1000)
+			alerts = s.pop_alerts()
+			a = alerts.pop(0)
+			self.assertTrue(isinstance(a, lt.log_alert))
+		# then the actual stats values
+		if len(alerts) == 0:
+			s.wait_for_alert(1000)
+			alerts = s.pop_alerts()
+		a = alerts.pop(0)
 		self.assertTrue(isinstance(a, lt.session_stats_alert))
 		self.assertTrue(isinstance(a.values, dict))
 		self.assertTrue(len(a.values) > 0)

docs/tuning.rst

@@ -23,6 +23,38 @@ computer.
 This document describes techniques to benchmark libtorrent performance
 and how parameters are likely to affect it.
 
+profiling
+=========
+
+libtorrent is instrumented with a number of counters and gauges you can have
+access to via the ``session_stats_alert``. First, enable these alerts in the
+alert mask::
+
+	settings_pack p;
+	p.set_int(settings_mask::alert_mask, alert::stats_notification);
+	ses.apply_settings(p);
+
+Then print alerts to a file::
+
+	std::vector<alert*> alerts;
+	ses.pop_alerts(&alerts);
+
+	for (auto* a : alerts) {
+		std::cout << a->message() << "\n";
+	}
+
+If you want to separate generic alerts from session stats, you can filter on the
+alert category in the alert, ``alert::category()``.
+
+The alerts with data will have the type ``session_stats_alert`` and there is one
+``session_log_alert`` that will be posted on startup containing the column names
+for all metrics. Logging this line will greatly simplify interpreting the output.
+
+The python scrip in ``tools/parse_session_stats.py`` can parse the resulting
+file and produce graphs of relevant stats. It requires gnuplot__.
+
+__ http://www.gnuplot.info
+
 reducing memory footprint
 =========================
 
@@ -145,21 +177,6 @@ all peers. This is the least amount of memory possible for the send buffer.
 You should benchmark your max send rate when adjusting this setting. If you have
 a very fast disk, you are less likely see a performance hit.
 
-optimize hashing for memory usage
----------------------------------
-
-When libtorrent is doing hash checks of a file, or when it re-reads a piece that
-was just completed to verify its hash, there are two options. The default one
-is optimized for speed, which allocates buffers for the entire piece, reads in
-the whole piece in one read call, then hashes it.
-
-The second option is to optimize for memory usage instead, where a single buffer
-is allocated, and the piece is read one block at a time, hashing it as each
-block is read from the file. For low memory environments, this latter approach
-is recommended. Change this by settings ``settings_pack::optimize_hashing_for_speed``
-to false. This will significantly reduce peak memory usage, especially for
-torrents with very large pieces.
-
 reduce executable size
 ----------------------
 
@@ -185,28 +202,6 @@ For all available options, see the `building libtorrent`_ secion.
 
 .. _`building libtorrent`: building.html
 
-play nice with the disk
-=======================
-
-When checking a torrent, libtorrent will try to read as fast as possible from the disk.
-The only thing that might hold it back is a CPU that is slow at calculating SHA-1 hashes,
-but typically the file checking is limited by disk read speed. Most operating systems
-today do not prioritize disk access based on the importance of the operation, this means
-that checking a torrent might delay other disk accesses, such as virtual memory swapping
-or just loading file by other (interactive) applications.
-
-In order to play nicer with the disk, and leave some spare time for it to service other
-processes that might be of higher importance to the end-user, you can introduce a sleep
-between the disc accesses. This is a direct tradeoff between how fast you can check a
-torrent and how soft you will hit the disk.
-
-You control this by setting the ``settings_pack::file_checks_delay_per_block`` to greater
-than zero. This number is the number of milliseconds to sleep between each read of 16 kiB.
-
-The sleeps are not necessarily in between each 16 kiB block (it might be read in larger chunks),
-but the number will be multiplied by the number of blocks that were read, to maintain the
-same semantics.
-
 high performance seeding
 ========================
 
@@ -251,25 +246,6 @@ the read cache is removed immediately. This saves a significant amount of cache
 which can be used as read-ahead for other peers. To enable volatile read cache, set
 ``settings_pack::volatile_read_cache`` to true.
 
-SSD as level 2 cache
---------------------
-
-It is possible to introduce a second level of cache, below the RAM disk cache. This is done
-by setting ``settings_pack::mmap_cache`` to a file path pointing to the SSD drive, and
-increasing the ``settings_pack::cache_size`` to the number of 16 kiB blocks would fit
-on the drive (or less).
-
-This will allocate disk buffers (for reading and writing) from a memory region that has
-been mapped to the specified file. If the drive this file lives on is not significantly
-faster than the destination drive, performance will be degraded. The point is to take
-advantage primarily of the fast read speed from SSD drives and use it to extend the read
-cache, improving seed performance.
-
-Which parts of the cache that actually live in RAM is determined by the operating system.
-
-Note that when using this feature, any block which ends up being pulled from the mmapped
-file will be considered a cache hit.
-
 uTP-TCP mixed mode
 ------------------
 
@@ -305,8 +281,8 @@ peers
 -----
 
 First of all, in order to allow many connections, set the global connection limit
-high, ``session::set_max_connections()``. Also set the upload rate limit to
+high, ``settings_pack::connections_limit``. Also set the upload rate limit to
-infinite, ``session::set_upload_rate_limit()``, passing 0 means infinite.
+infinite, ``settings_pack::upload_rate_limit``, 0 means infinite.
 
 When dealing with a large number of peers, it might be a good idea to have slightly
 stricter timeouts, to get rid of lingering connections as soon as possible.
@@ -319,9 +295,10 @@ multiple connections from the same IP. That way two people from behind the same
 can use the service simultaneously. This is controlled by
 ``settings_pack::allow_multiple_connections_per_ip``.
 
-In order to always unchoke peers, turn off automatic unchoke
+In order to always unchoke peers, turn off automatic unchoke by setting
-``settings_pack::auto_upload_slots`` and set the number of upload slots to a large
+``settings_pack::choking_algorithm`` to ``fixed_slot_choker`` and set the number
-number via ``session::set_max_uploads()``, or use -1 (which means infinite).
+of upload slots to a large number via ``settings_pack::unchoke_slots_limit``,
+or use -1 (which means infinite).
 
 torrent limits
 --------------
@@ -356,12 +333,12 @@ the returned vector. If you have a lot of torrents, you might want to update the
 of only certain torrents. For instance, you might only be interested in torrents that
 are being downloaded.
 
-The intended use of these functions is to start off by calling ``get_torrent_status``
+The intended use of these functions is to start off by calling ``get_torrent_status()``
-to get a list of all torrents that match your criteria. Then call ``refresh_torrent_status``
+to get a list of all torrents that match your criteria. Then call ``refresh_torrent_status()``
 on that list. This will only refresh the status for the torrents in your list, and thus
 ignore all other torrents you might be running. This may save a significant amount of
 time, especially if the number of torrents you're interested in is small. In order to
-keep your list of interested torrents up to date, you can either call ``get_torrent_status``
+keep your list of interested torrents up to date, you can either call ``get_torrent_status()``
 from time to time, to include torrents you might have become interested in since the last
 time. In order to stop refreshing a certain torrent, simply remove it from the list.
 
@@ -370,6 +347,9 @@ update your list based on these alerts. There are alerts for when torrents are a
 paused, resumed, completed etc. Doing this ensures that you only query status for the
 minimal set of torrents you are actually interested in.
 
+To get an update with only the torrents that have changed since last time, call
+``session::post_torrent_updates()``.
+
 benchmarking
 ============
 
@@ -455,7 +435,7 @@ covered here, or if you have improved any of the parser scrips, please consider
 contributing it back to the project.
 
 If you have run tests and found that some algorithm or default value in
-libtorrent is suboptimal, please contribute that knowledge back as well, to
+libtorrent are suboptimal, please contribute that knowledge back as well, to
 allow us to improve the library.
 
 If you have additional suggestions on how to tune libtorrent for any specific

include/libtorrent/alert_types.hpp

@@ -1559,6 +1559,8 @@ namespace libtorrent
 	// This alert is posted approximately once every second, and it contains
 	// byte counters of most statistics that's tracked for torrents. Each active
 	// torrent posts these alerts regularly.
+	// This alert has been superceded by calling ``post_torrent_updates()``
+	// regularly on the session object. This alert will be removed
 	struct TORRENT_EXPORT stats_alert TORRENT_FINAL : torrent_alert
 	{
 		// internal
@@ -2448,7 +2450,7 @@ namespace libtorrent
 	// this is posted when one or more blocks are picked by the piece picker,
 	// assuming the verbose piece picker logging is enabled (see
 	// picker_log_notification).
-	struct TORRENT_EXPORT picker_log_alert : peer_alert
+	struct TORRENT_EXPORT picker_log_alert TORRENT_FINAL : peer_alert
 	{
 #ifndef TORRENT_DISABLE_LOGGING
 

include/libtorrent/settings_pack.hpp

@@ -252,10 +252,10 @@ namespace libtorrent
 			// in a swarm has the same IP address.
 			allow_multiple_connections_per_ip = bool_type_base,
 
+#ifndef TORRENT_NO_DEPRECATE
 			// if set to true, upload, download and unchoke limits are ignored for
 			// peers on the local network. This option is *DEPRECATED*, please use
 			// set_peer_class_filter() instead.
-#ifndef TORRENT_NO_DEPRECATE
 			ignore_limits_on_local_network,
 #else
 			deprecated1,
@@ -930,7 +930,7 @@ namespace libtorrent
 			//   still allocates all upload capacity, but shuffles it around to
 			//   the best peers first. For this choker to be efficient, you need
 			//   to set a global upload rate limit
-			//   (``session::set_upload_rate_limit()``). For more information
+			//   (``settings_pack::upload_rate_limit``). For more information
 			//   about this choker, see the paper_. This choker is not fully
 			//   implemented nor tested.
 			// 
@@ -1170,6 +1170,7 @@ namespace libtorrent
 			recv_socket_buffer_size,
 			send_socket_buffer_size,
 
+#ifndef TORRENT_NO_DEPRECATE
 			// ``file_checks_delay_per_block`` is the number of milliseconds to
 			// sleep in between disk read operations when checking torrents. This
 			// defaults to 0, but can be set to higher numbers to slow down the
@@ -1178,6 +1179,9 @@ namespace libtorrent
 			// bit longer, as long as they leave disk I/O time for other
 			// processes.
 			file_checks_delay_per_block,
+#else
+			deprecated14,
+#endif
 
 			// ``read_cache_line_size`` is the number of blocks to read into the
 			// read cache when a read cache miss occurs. Setting this to 0 is
@@ -1287,20 +1291,11 @@ namespace libtorrent
 			// share_mode_target is set to more than 3, nothing is downloaded.
 			share_mode_target,
 
-			// ``upload_rate_limit``, ``download_rate_limit``,
+			// ``upload_rate_limit`` and ``download_rate_limit`` sets
-			// ``local_upload_rate_limit`` and ``local_download_rate_limit`` sets
 			// the session-global limits of upload and download rate limits, in
-			// bytes per second. The local rates refer to peers on the local
+			// bytes per second. By default peers on the local network are not rate
-			// network. By default peers on the local network are not rate
 			// limited.
 			// 
-			// These rate limits are only used for local peers (peers within the
-			// same subnet as the client itself) and it is only used when
-			// ``ignore_limits_on_local_network`` is set to true (which it is by
-			// default). These rate limits default to unthrottled, but can be
-			// useful in case you want to treat local peers preferentially, but
-			// not quite unthrottled.
-			// 
 			// A value of 0 means unlimited.
 			upload_rate_limit,
 			download_rate_limit,

src/session.cpp

@@ -117,11 +117,6 @@ namespace libtorrent
 		// connect to 5 peers per second
 		set.set_int(settings_pack::connection_speed, 5);
 
-		// be extra nice on the hard drive when running
-		// on embedded devices. This might slow down
-		// torrent checking
-		set.set_int(settings_pack::file_checks_delay_per_block, 5);
-
 		// only have 4 files open at a time
 		set.set_int(settings_pack::file_pool_size, 4);
 

src/session_impl.cpp

@@ -566,7 +566,8 @@ namespace aux {
 		TORRENT_ASSERT(is_single_thread());
 
 #ifndef TORRENT_DISABLE_LOGGING
-		if (m_alerts.should_post<log_alert>())
+		if (m_alerts.should_post<log_alert>()
+			|| m_alerts.should_post<session_stats_alert>())
 		{
 			session_log(" *** session thread init");
 

src/settings_pack.cpp

@@ -285,7 +285,7 @@ namespace libtorrent
 		SET(max_rejects, 50, 0),
 		SET(recv_socket_buffer_size, 0, &session_impl::update_socket_buffer_size),
 		SET(send_socket_buffer_size, 0, &session_impl::update_socket_buffer_size),
-		SET(file_checks_delay_per_block, 0, 0),
+		DEPRECATED_SET(file_checks_delay_per_block, 0, 0),
 		SET(read_cache_line_size, 32, 0),
 		SET(write_cache_line_size, 16, 0),
 		SET(optimistic_disk_retry, 10 * 60, 0),