diff --git a/ChangeLog b/ChangeLog index bcf909216..629292489 100644 --- a/ChangeLog +++ b/ChangeLog @@ -56,6 +56,7 @@ * resume data no longer has timestamps of files * require C++11 to build libtorrent + * fix ABI compatibility issue introduced with preformatted entry type * add web_seed_name_lookup_retry to session_settings * slightly improve proxy settings backwards compatibility * add function to get default settings diff --git a/appveyor.yml b/appveyor.yml index 52e4f02f2..e9a04d5ef 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -52,7 +52,7 @@ install: - if not defined linkflags ( set linkflags="" ) - if not defined include ( set include="" ) - cd %ROOT_DIRECTORY% -- set BOOST_ROOT=c:\Libraries\boost_1_59_0 +- set BOOST_ROOT=c:\Libraries\boost_1_63_0 - set BOOST_BUILD_PATH=%BOOST_ROOT%\tools\build - echo %BOOST_ROOT% - echo %BOOST_BUILD_PATH% diff --git a/bindings/python/test.py b/bindings/python/test.py index 533ca4b70..b615b4aed 100644 --- a/bindings/python/test.py +++ b/bindings/python/test.py @@ -13,6 +13,7 @@ import shutil import binascii import subprocess as sub import sys +import inspect # include terminal interface for travis parallel executions of scripts which use # terminal features: fix multiple stdin assignment at termios.tcgetattr @@ -365,11 +366,25 @@ class test_session(unittest.TestCase): def test_post_session_stats(self): s = lt.session({'alert_mask': lt.alert.category_t.stats_notification, 'enable_dht': False}) - s.post_session_stats() - a = s.wait_for_alert(1000) - self.assertTrue(isinstance(a, lt.session_stats_alert)) - self.assertTrue(isinstance(a.values, dict)) - self.assertTrue(len(a.values) > 0) + + def test_post_session_stats(self): + s = lt.session({'alert_mask': lt.alert.category_t.stats_notification, 'enable_dht': False}) + s.post_session_stats() + alerts = [] + # first the stats headers log line. but not if logging is disabled + if 'log_alert' in [i[0] for i in inspect.getmembers(lt)]: + s.wait_for_alert(1000) + alerts = s.pop_alerts() + a = alerts.pop(0) + self.assertTrue(isinstance(a, lt.log_alert)) + # then the actual stats values + if len(alerts) == 0: + s.wait_for_alert(1000) + alerts = s.pop_alerts() + a = alerts.pop(0) + self.assertTrue(isinstance(a, lt.session_stats_alert)) + self.assertTrue(isinstance(a.values, dict)) + self.assertTrue(len(a.values) > 0) def test_unknown_settings(self): try: diff --git a/docs/tuning.rst b/docs/tuning.rst index b9f97664d..bc15be3f2 100644 --- a/docs/tuning.rst +++ b/docs/tuning.rst @@ -23,6 +23,38 @@ computer. This document describes techniques to benchmark libtorrent performance and how parameters are likely to affect it. +profiling +========= + +libtorrent is instrumented with a number of counters and gauges you can have +access to via the ``session_stats_alert``. First, enable these alerts in the +alert mask:: + + settings_pack p; + p.set_int(settings_mask::alert_mask, alert::stats_notification); + ses.apply_settings(p); + +Then print alerts to a file:: + + std::vector alerts; + ses.pop_alerts(&alerts); + + for (auto* a : alerts) { + std::cout << a->message() << "\n"; + } + +If you want to separate generic alerts from session stats, you can filter on the +alert category in the alert, ``alert::category()``. + +The alerts with data will have the type ``session_stats_alert`` and there is one +``session_log_alert`` that will be posted on startup containing the column names +for all metrics. Logging this line will greatly simplify interpreting the output. + +The python scrip in ``tools/parse_session_stats.py`` can parse the resulting +file and produce graphs of relevant stats. It requires gnuplot__. + +__ http://www.gnuplot.info + reducing memory footprint ========================= @@ -145,21 +177,6 @@ all peers. This is the least amount of memory possible for the send buffer. You should benchmark your max send rate when adjusting this setting. If you have a very fast disk, you are less likely see a performance hit. -optimize hashing for memory usage ---------------------------------- - -When libtorrent is doing hash checks of a file, or when it re-reads a piece that -was just completed to verify its hash, there are two options. The default one -is optimized for speed, which allocates buffers for the entire piece, reads in -the whole piece in one read call, then hashes it. - -The second option is to optimize for memory usage instead, where a single buffer -is allocated, and the piece is read one block at a time, hashing it as each -block is read from the file. For low memory environments, this latter approach -is recommended. Change this by settings ``settings_pack::optimize_hashing_for_speed`` -to false. This will significantly reduce peak memory usage, especially for -torrents with very large pieces. - reduce executable size ---------------------- @@ -185,28 +202,6 @@ For all available options, see the `building libtorrent`_ secion. .. _`building libtorrent`: building.html -play nice with the disk -======================= - -When checking a torrent, libtorrent will try to read as fast as possible from the disk. -The only thing that might hold it back is a CPU that is slow at calculating SHA-1 hashes, -but typically the file checking is limited by disk read speed. Most operating systems -today do not prioritize disk access based on the importance of the operation, this means -that checking a torrent might delay other disk accesses, such as virtual memory swapping -or just loading file by other (interactive) applications. - -In order to play nicer with the disk, and leave some spare time for it to service other -processes that might be of higher importance to the end-user, you can introduce a sleep -between the disc accesses. This is a direct tradeoff between how fast you can check a -torrent and how soft you will hit the disk. - -You control this by setting the ``settings_pack::file_checks_delay_per_block`` to greater -than zero. This number is the number of milliseconds to sleep between each read of 16 kiB. - -The sleeps are not necessarily in between each 16 kiB block (it might be read in larger chunks), -but the number will be multiplied by the number of blocks that were read, to maintain the -same semantics. - high performance seeding ======================== @@ -251,25 +246,6 @@ the read cache is removed immediately. This saves a significant amount of cache which can be used as read-ahead for other peers. To enable volatile read cache, set ``settings_pack::volatile_read_cache`` to true. -SSD as level 2 cache --------------------- - -It is possible to introduce a second level of cache, below the RAM disk cache. This is done -by setting ``settings_pack::mmap_cache`` to a file path pointing to the SSD drive, and -increasing the ``settings_pack::cache_size`` to the number of 16 kiB blocks would fit -on the drive (or less). - -This will allocate disk buffers (for reading and writing) from a memory region that has -been mapped to the specified file. If the drive this file lives on is not significantly -faster than the destination drive, performance will be degraded. The point is to take -advantage primarily of the fast read speed from SSD drives and use it to extend the read -cache, improving seed performance. - -Which parts of the cache that actually live in RAM is determined by the operating system. - -Note that when using this feature, any block which ends up being pulled from the mmapped -file will be considered a cache hit. - uTP-TCP mixed mode ------------------ @@ -305,8 +281,8 @@ peers ----- First of all, in order to allow many connections, set the global connection limit -high, ``session::set_max_connections()``. Also set the upload rate limit to -infinite, ``session::set_upload_rate_limit()``, passing 0 means infinite. +high, ``settings_pack::connections_limit``. Also set the upload rate limit to +infinite, ``settings_pack::upload_rate_limit``, 0 means infinite. When dealing with a large number of peers, it might be a good idea to have slightly stricter timeouts, to get rid of lingering connections as soon as possible. @@ -319,9 +295,10 @@ multiple connections from the same IP. That way two people from behind the same can use the service simultaneously. This is controlled by ``settings_pack::allow_multiple_connections_per_ip``. -In order to always unchoke peers, turn off automatic unchoke -``settings_pack::auto_upload_slots`` and set the number of upload slots to a large -number via ``session::set_max_uploads()``, or use -1 (which means infinite). +In order to always unchoke peers, turn off automatic unchoke by setting +``settings_pack::choking_algorithm`` to ``fixed_slot_choker`` and set the number +of upload slots to a large number via ``settings_pack::unchoke_slots_limit``, +or use -1 (which means infinite). torrent limits -------------- @@ -356,12 +333,12 @@ the returned vector. If you have a lot of torrents, you might want to update the of only certain torrents. For instance, you might only be interested in torrents that are being downloaded. -The intended use of these functions is to start off by calling ``get_torrent_status`` -to get a list of all torrents that match your criteria. Then call ``refresh_torrent_status`` +The intended use of these functions is to start off by calling ``get_torrent_status()`` +to get a list of all torrents that match your criteria. Then call ``refresh_torrent_status()`` on that list. This will only refresh the status for the torrents in your list, and thus ignore all other torrents you might be running. This may save a significant amount of time, especially if the number of torrents you're interested in is small. In order to -keep your list of interested torrents up to date, you can either call ``get_torrent_status`` +keep your list of interested torrents up to date, you can either call ``get_torrent_status()`` from time to time, to include torrents you might have become interested in since the last time. In order to stop refreshing a certain torrent, simply remove it from the list. @@ -370,6 +347,9 @@ update your list based on these alerts. There are alerts for when torrents are a paused, resumed, completed etc. Doing this ensures that you only query status for the minimal set of torrents you are actually interested in. +To get an update with only the torrents that have changed since last time, call +``session::post_torrent_updates()``. + benchmarking ============ @@ -428,7 +408,7 @@ covered here, or if you have improved any of the parser scrips, please consider contributing it back to the project. If you have run tests and found that some algorithm or default value in -libtorrent is suboptimal, please contribute that knowledge back as well, to +libtorrent are suboptimal, please contribute that knowledge back as well, to allow us to improve the library. If you have additional suggestions on how to tune libtorrent for any specific diff --git a/include/libtorrent/alert_types.hpp b/include/libtorrent/alert_types.hpp index e2d1a274e..6281d5873 100644 --- a/include/libtorrent/alert_types.hpp +++ b/include/libtorrent/alert_types.hpp @@ -1558,6 +1558,8 @@ namespace libtorrent // This alert is posted approximately once every second, and it contains // byte counters of most statistics that's tracked for torrents. Each active // torrent posts these alerts regularly. + // This alert has been superceded by calling ``post_torrent_updates()`` + // regularly on the session object. This alert will be removed struct TORRENT_EXPORT stats_alert final : torrent_alert { // internal @@ -2440,7 +2442,7 @@ namespace libtorrent // this is posted when one or more blocks are picked by the piece picker, // assuming the verbose piece picker logging is enabled (see // picker_log_notification). - struct TORRENT_EXPORT picker_log_alert : peer_alert + struct TORRENT_EXPORT picker_log_alert final : peer_alert { // internal picker_log_alert(aux::stack_allocator& alloc, torrent_handle const& h diff --git a/include/libtorrent/create_torrent.hpp b/include/libtorrent/create_torrent.hpp index 38b2b4670..f1ed93a90 100644 --- a/include/libtorrent/create_torrent.hpp +++ b/include/libtorrent/create_torrent.hpp @@ -40,6 +40,8 @@ POSSIBILITY OF SUCH DAMAGE. #include "libtorrent/hasher.hpp" #include "libtorrent/string_view.hpp" #include "libtorrent/aux_/vector.hpp" +#include "libtorrent/file.hpp" // for combine_path etc. +#include "libtorrent/version.hpp" #include #include diff --git a/include/libtorrent/settings_pack.hpp b/include/libtorrent/settings_pack.hpp index adce55ed5..ff11894cf 100644 --- a/include/libtorrent/settings_pack.hpp +++ b/include/libtorrent/settings_pack.hpp @@ -251,10 +251,10 @@ namespace libtorrent // in a swarm has the same IP address. allow_multiple_connections_per_ip = bool_type_base, +#ifndef TORRENT_NO_DEPRECATE // if set to true, upload, download and unchoke limits are ignored for // peers on the local network. This option is *DEPRECATED*, please use // set_peer_class_filter() instead. -#ifndef TORRENT_NO_DEPRECATE ignore_limits_on_local_network, #else deprecated1, @@ -946,7 +946,7 @@ namespace libtorrent // still allocates all upload capacity, but shuffles it around to // the best peers first. For this choker to be efficient, you need // to set a global upload rate limit - // (``session::set_upload_rate_limit()``). For more information + // (``settings_pack::upload_rate_limit``). For more information // about this choker, see the paper_. This choker is not fully // implemented nor tested. // @@ -1179,6 +1179,7 @@ namespace libtorrent // allowed to grow to. max_peer_recv_buffer_size, +#ifndef TORRENT_NO_DEPRECATE // ``file_checks_delay_per_block`` is the number of milliseconds to // sleep in between disk read operations when checking torrents. This // defaults to 0, but can be set to higher numbers to slow down the @@ -1187,6 +1188,9 @@ namespace libtorrent // bit longer, as long as they leave disk I/O time for other // processes. file_checks_delay_per_block, +#else + deprecated14, +#endif // ``read_cache_line_size`` is the number of blocks to read into the // read cache when a read cache miss occurs. Setting this to 0 is @@ -1300,20 +1304,11 @@ namespace libtorrent // share_mode_target is set to more than 3, nothing is downloaded. share_mode_target, - // ``upload_rate_limit``, ``download_rate_limit``, - // ``local_upload_rate_limit`` and ``local_download_rate_limit`` sets + // ``upload_rate_limit`` and ``download_rate_limit`` sets // the session-global limits of upload and download rate limits, in - // bytes per second. The local rates refer to peers on the local - // network. By default peers on the local network are not rate + // bytes per second. By default peers on the local network are not rate // limited. - // - // These rate limits are only used for local peers (peers within the - // same subnet as the client itself) and it is only used when - // ``ignore_limits_on_local_network`` is set to true (which it is by - // default). These rate limits default to unthrottled, but can be - // useful in case you want to treat local peers preferentially, but - // not quite unthrottled. - // + // // A value of 0 means unlimited. upload_rate_limit, download_rate_limit, diff --git a/src/create_torrent.cpp b/src/create_torrent.cpp index 36c634aa7..058f5878c 100644 --- a/src/create_torrent.cpp +++ b/src/create_torrent.cpp @@ -368,7 +368,7 @@ namespace libtorrent create_torrent::create_torrent(torrent_info const& ti) : m_files(const_cast(ti.files())) - , m_creation_date(time(nullptr)) + , m_creation_date(time(0)) , m_multifile(ti.num_files() > 1) , m_private(ti.priv()) , m_merkle_torrent(ti.is_merkle_torrent()) diff --git a/src/enum_net.cpp b/src/enum_net.cpp index 70eda3f2f..75a268e13 100644 --- a/src/enum_net.cpp +++ b/src/enum_net.cpp @@ -677,7 +677,7 @@ namespace libtorrent for (;i != udp::resolver_iterator(); ++i) { iface.interface_address = i->endpoint().address(); - iface.name[0] = 0; + iface.name[0] = '\0'; iface.mtu = 1500; if (iface.interface_address.is_v4()) iface.netmask = address_v4::netmask(iface.interface_address.to_v4()); diff --git a/src/session.cpp b/src/session.cpp index acae59767..7f31dcfe5 100644 --- a/src/session.cpp +++ b/src/session.cpp @@ -79,11 +79,6 @@ namespace libtorrent // connect to 5 peers per second set.set_int(settings_pack::connection_speed, 5); - // be extra nice on the hard drive when running - // on embedded devices. This might slow down - // torrent checking - set.set_int(settings_pack::file_checks_delay_per_block, 5); - // only have 4 files open at a time set.set_int(settings_pack::file_pool_size, 4); diff --git a/src/session_impl.cpp b/src/session_impl.cpp index ded366e45..7a3d0b7e8 100644 --- a/src/session_impl.cpp +++ b/src/session_impl.cpp @@ -588,12 +588,11 @@ namespace aux { TORRENT_ASSERT(is_single_thread()); #ifndef TORRENT_DISABLE_LOGGING - // this alert is a bit special. Since it's so verbose it's not only - // filtered by its own alert type (log_alert) but also whether session - // stats alerts are actually enabled. Without session_stats alerts the - // headers aren't very useful anyway + // this alert is a bit special. The stats headers aren't very useful + // unless session_stats is enabled, sp it's posted in the session_Stats + // category as well if (m_alerts.should_post() - && m_alerts.should_post()) + || m_alerts.should_post()) { session_log(" *** session thread init"); diff --git a/src/settings_pack.cpp b/src/settings_pack.cpp index cbc0b91cc..6110a1287 100644 --- a/src/settings_pack.cpp +++ b/src/settings_pack.cpp @@ -262,7 +262,7 @@ namespace libtorrent SET(recv_socket_buffer_size, 0, &session_impl::update_socket_buffer_size), SET(send_socket_buffer_size, 0, &session_impl::update_socket_buffer_size), SET(max_peer_recv_buffer_size, 2 * 1024 * 1024, nullptr), - SET(file_checks_delay_per_block, 0, nullptr), + DEPRECATED_SET(file_checks_delay_per_block, 0, nullptr), SET(read_cache_line_size, 32, nullptr), SET(write_cache_line_size, 16, nullptr), SET(optimistic_disk_retry, 10 * 60, nullptr), diff --git a/src/torrent.cpp b/src/torrent.cpp index b3d21c1ce..1db100f90 100644 --- a/src/torrent.cpp +++ b/src/torrent.cpp @@ -5980,9 +5980,14 @@ namespace libtorrent { if (m_magnet_link || (m_save_resume_flags & torrent_handle::save_info_dict)) { - boost::shared_array const info = torrent_file().metadata(); - int const size = torrent_file().metadata_size(); - ret["info"].preformatted().assign(&info[0], &info[0] + size); + ret["info"] = bdecode(&torrent_file().metadata()[0] + , &torrent_file().metadata()[0] + torrent_file().metadata_size()); +// TODO: re-enable this code once there's a non-inlined encoder function. Or +// perhaps this should not be used until saving resume_data via +// add_torrent_params and a free function, similar to read_resume_data +// boost::shared_array const info = torrent_file().metadata(); +// int const size = torrent_file().metadata_size(); +// ret["info"].preformatted().assign(&info[0], &info[0] + size); } } diff --git a/test/main.cpp b/test/main.cpp index 4b2a94bfe..641be329b 100644 --- a/test/main.cpp +++ b/test/main.cpp @@ -47,6 +47,7 @@ POSSIBILITY OF SUCH DAMAGE. #include "libtorrent/assert.hpp" #include "libtorrent/file.hpp" #include "libtorrent/random.hpp" +#include "libtorrent/aux_/escape_string.hpp" #include #ifdef _WIN32 @@ -217,6 +218,51 @@ void print_usage(char const* executable) "by -l. If no test is specified, all tests are run\n", executable); } +void change_directory(std::string const& f, error_code& ec) +{ + ec.clear(); + +#ifdef TORRENT_WINDOWS +#if TORRENT_USE_WSTRING +#define SetCurrentDirectory_ SetCurrentDirectoryW + std::wstring n = convert_to_wstring(f); +#else +#define SetCurrentDirectory_ SetCurrentDirectoryA + std::string const& n = convert_to_native(f); +#endif // TORRENT_USE_WSTRING + + if (SetCurrentDirectory_(n.c_str()) == 0) + ec.assign(GetLastError(), system_category()); +#else + std::string n = convert_to_native(f); + int ret = ::chdir(n.c_str()); + if (ret != 0) + ec.assign(errno, system_category()); +#endif +} + +struct unit_directory_guard +{ + std::string dir; + unit_directory_guard(unit_directory_guard const&) = delete; + unit_directory_guard& operator=(unit_directory_guard const&) = delete; + ~unit_directory_guard() + { + if (keep_files) return; + error_code ec; + std::string const parent_dir = parent_path(dir); + // windows will not allow to remove current dir, so let's change it to root + change_directory(parent_dir, ec); + if (ec) + { + TEST_ERROR("Failed to change directory: " + ec.message()); + return; + } + remove_all(dir, ec); + if (ec) TEST_ERROR("Failed to remove unit test directory: " + ec.message()); + } +}; + EXPORT int main(int argc, char const* argv[]) { char const* executable = argv[0]; @@ -315,32 +361,12 @@ EXPORT int main(int argc, char const* argv[]) #else process_id = getpid(); #endif + std::string root_dir = current_working_directory(); char dir[40]; - std::snprintf(dir, sizeof(dir), "test_tmp_%u", process_id); - std::string test_dir = complete(dir); - error_code ec; - create_directory(test_dir, ec); - if (ec) - { - std::printf("Failed to create test directory: %s\n", ec.message().c_str()); - return 1; - } - int ret; -#ifdef TORRENT_WINDOWS - SetCurrentDirectoryA(dir); -#else - ret = chdir(dir); - if (ret != 0) - { - std::printf("failed to change directory to \"%s\": %s" - , dir, strerror(errno)); - return 1; - } -#endif - std::printf("test: %s\ncwd = \"%s\"\nrnd: %x\n" - , executable, test_dir.c_str(), libtorrent::random(0xffffffff)); - - int total_failures = 0; + snprintf(dir, sizeof(dir), "test_tmp_%u", process_id); + std::string unit_dir_prefix = combine_path(root_dir, dir); + std::printf("test: %s\ncwd_prefix = \"%s\"\nrnd = %x\n" + , executable, unit_dir_prefix.c_str(), libtorrent::random(0xffffffff)); if (_g_num_unit_tests == 0) { @@ -357,6 +383,25 @@ EXPORT int main(int argc, char const* argv[]) if (filter && tests_to_run.count(_g_unit_tests[i].name) == 0) continue; + std::string unit_dir = unit_dir_prefix; + char i_str[40]; + snprintf(i_str, sizeof(i_str), "%u", i); + unit_dir.append(i_str); + error_code ec; + create_directory(unit_dir, ec); + if (ec) + { + std::printf("Failed to create unit test directory: %s\n", ec.message().c_str()); + return 1; + } + unit_directory_guard unit_dir_guard{unit_dir}; + change_directory(unit_dir, ec); + if (ec) + { + std::printf("Failed to change unit test directory: %s\n", ec.message().c_str()); + return 1; + } + unit_test_t& t = _g_unit_tests[i]; if (redirect_stdout || redirect_stderr) @@ -399,7 +444,6 @@ EXPORT int main(int argc, char const* argv[]) try { #endif - _g_test_failures = 0; (*t.fun)(); #ifndef BOOST_NO_EXCEPTIONS @@ -434,7 +478,6 @@ EXPORT int main(int argc, char const* argv[]) t.num_failures = _g_test_failures; t.run = true; - total_failures += _g_test_failures; ++num_run; if (redirect_stdout && t.output) @@ -471,16 +514,6 @@ EXPORT int main(int argc, char const* argv[]) if (redirect_stdout) fflush(stdout); if (redirect_stderr) fflush(stderr); - ret = print_failures(); -#if !defined TORRENT_LOGGING - if (ret == 0 && !keep_files) - { - remove_all(test_dir, ec); - if (ec) - std::printf("failed to remove test dir: %s\n", ec.message().c_str()); - } -#endif - - return total_failures ? 333 : 0; + return print_failures() ? 333 : 0; } diff --git a/test/test.cpp b/test/test.cpp index ba224b21a..c60c2fa43 100644 --- a/test/test.cpp +++ b/test/test.cpp @@ -36,7 +36,7 @@ POSSIBILITY OF SUCH DAMAGE. unit_test_t _g_unit_tests[1024]; int _g_num_unit_tests = 0; -int _g_test_failures = 0; +int _g_test_failures = 0; // flushed at start of every unit int _g_test_idx = 0; static std::vector failure_strings; @@ -65,6 +65,7 @@ int print_failures() } std::printf("\n\n"); + int total_num_failures = 0; for (int i = 0; i < _g_num_unit_tests; ++i) { @@ -77,6 +78,7 @@ int print_failures() } else { + total_num_failures += _g_unit_tests[i].num_failures; std::printf("\x1b[31m[%-*s] %d FAILURES\n" , longest_name , _g_unit_tests[i].name @@ -86,8 +88,8 @@ int print_failures() std::printf("\x1b[0m"); - if (_g_test_failures > 0) - std::printf("\n\n\x1b[41m == %d TEST(S) FAILED ==\x1b[0m\n\n\n", _g_test_failures); - return _g_test_failures; + if (total_num_failures > 0) + std::printf("\n\n\x1b[41m == %d TEST(S) FAILED ==\x1b[0m\n\n\n", total_num_failures); + return total_num_failures; }