From dab0f8b8d0559a3bfa6c3b4e6490de5ee70b83a6 Mon Sep 17 00:00:00 2001 From: arvidn Date: Sat, 19 Sep 2015 17:49:01 -0400 Subject: [PATCH] simplify the queuing logic for checking torrents. make all non-auto-managed torrents always be exempt from any queuing mechanism (including checking). Extend documentation on how it works --- ChangeLog | 2 + docs/manual.rst | 195 +++++++++++++++++++------- include/libtorrent/settings_pack.hpp | 10 +- include/libtorrent/torrent.hpp | 19 ++- include/libtorrent/torrent_handle.hpp | 10 +- src/session_impl.cpp | 59 ++++---- src/settings_pack.cpp | 1 + src/torrent.cpp | 81 ++++++----- 8 files changed, 248 insertions(+), 129 deletions(-) diff --git a/ChangeLog b/ChangeLog index b3020cfe3..203c640bf 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,5 @@ + * make all non-auto managed torrents exempt from queuing logic, including + checking torrents. * add option to not proxy tracker connections through proxy * removed sparse-regions feature * support using 0 disk threads (to perform disk I/O in network thread) diff --git a/docs/manual.rst b/docs/manual.rst index 95bacaca7..d52b1ae4d 100644 --- a/docs/manual.rst +++ b/docs/manual.rst @@ -174,20 +174,130 @@ The format of the magnet URI is: queuing ======= -libtorrent supports *queuing*. Which means it makes sure that a limited number of -torrents are being downloaded at any given time, and once a torrent is completely -downloaded, the next in line is started. +libtorrent supports *queuing*. Quing is a mechanism to automatically pause and +resume torrents based on certain criteria. The criteria depends on the overall +state the torrent is in (checking, downloading or seeding). -Torrents that are *auto managed* are subject to the queuing and the active -torrents limits. To make a torrent auto managed, set add_torrent_params::flag_auto_managed -when adding the torrent (see async_add_torrent() and add_torrent()). +To opt-out of the queuing logic, make sure your torrents are added with the +add_torrent_params::flag_auto_managed bit *cleared*. Or call +torrent_handle::auto_managed() passing false on the torrent handle. -The limits of the number of downloading and seeding torrents are controlled via -settings_pack::active_downloads, settings_pack::active_seeds and settings_pack::active_limit in -settings_pack. These limits takes non auto managed torrents into account as -well. If there are more non-auto managed torrents being downloaded than the -settings_pack::active_downloads setting, any auto managed torrents will be queued until -torrents are removed so that the number drops below the limit. +The overall purpose of the queuing logic is to improve performance under arbitrary +torrent downloading and seeding load. For example, if you want to download 100 +torrents on a limited home connection, you improve performance by downloading +them one at a time (or maybe two at a time), over downloading them all in +parallel. The benefits are: + +* the average completion time of a torrent is half of what it would be if all + downloaded in parallel. +* The amount of upload capacity is more likely to reach the *reciprocation rate* + of your peers, and is likely to improve your *return on investment* (download + to upload ratio) +* your disk I/O load is likely to be more local which may improve I/O + performance and decrease fragmentation. + +There are fundamentally 3 seaparate queues: + +* checking torrents +* downloading torrents +* seeding torrents + +Every torrent that is not seeding has a queue number associated with it, this is +its place in line to be started. See torrent_status::queue_position. + +On top of the limits of each queue, there is an over arching limit, set int +settings_pack::active_limit. The auto manager will never start more than this +number of torrents. Non-auto-managed torrents are exempt from this logic, and +not counted. + +At a regular interval, torrents are checked if there needs to be any +re-ordering of which torrents are active and which are queued. This interval +can be controlled via settings_pack::auto_manage_interval. + +For queuing to work, resume data needs to be saved and restored for all +torrents. See torrent_handle::save_resume_data(). + +queue position +-------------- + +The torrents in the front of the queue are started and the rest are ordered with +regards to their queue position. Any newly added torrent is placed at the end of +the queue. Once a torrent is removed or turns into a seed, its queue position is +-1 and all torrents that used to be after it in the queue, decreases their +position in order to fill the gap. + +The queue positions are always contiguous, in a sequence without any gaps. + +Lower queue position means closer to the front of the queue, and will be +started sooner than torrents with higher queue positions. + +To query a torrent for its position in the queue, or change its position, see: +torrent_handle::queue_position(), torrent_handle::queue_position_up(), +torrent_handle::queue_position_down(), torrent_handle::queue_position_top() +and torrent_handle::queue_position_bottom(). + +checking queue +-------------- + +The checking queue affects torrents in the torrent_status::checking or +torrent_status::allocating state that are auto-managed. + +The checking queue will make sure that (of the torrents in its queue) no more than +settings_pack::active_checking_limit torrents are started at any given time. +Once a torrent completes checking and moves into a diffferent state, the next in +line will be started for checking. + +Any torrent added force-started or force-stopped (i.e. the auto managed flag is +_not_ set), will not be subject to this limit and they will all check +independently and in parallel. + +downloading queue +----------------- + +Similarly to the checking queue, the downloading queue will make sure that no +more than settings_pack::active_downloads torrents are in the downloading +state at any given time. + +The torrent_status::queue_position is used again here to determine who is next +in line to be started once a downloading torrent completes or is stopped/removed. + +seeding queue +------------- + +The seeding queue does not use torrent_status::queue_position to determine which +torrent to seed. Instead, it estimates the *demand* for the torrent to be +seeded. A torrent with few other seeds and many downloaders is assumed to have a +higher demand of more seeds than one with many seeds and few downloaders. + +It limits the number of started seeds to settings_pack::active_seeds. + +On top of this basic bias, *seed priority* can be controller by specifying a +seed ratio (the upload to download ratio), a seed-time ratio (the download +time to seeding time ratio) and a seed-time (the abosulte time to be seeding a +torrent). Until all those targets are hit, the torrent will be prioritized for +seeding. + +Among torrents that have met their seed target, torrents where we don't know of +any other seed take strict priority. + +In order to avoid flapping, torrents that were started less than 30 minutes ago +also have priority to keep seeding. + +Finally, for torrents where none of the above apply, they are prioritized based +on the download to seed ratio. + +The relevant settings to control these limits are +settings_pack::share_ratio_limit, settings_pack::seed_time_ratio_limit and +settings_pack::seed_time_limit. + +queuing options +--------------- + +In addition to simply starting and stopping torrents, the queuing mechanism can +be more fine grained in its control of the resources used by torrents. + +half-started torrents +..................... In addition to the downloading and seeding limits, there are limits on *actions* torrents perform. The downloading and seeding limits control whether peers are @@ -203,49 +313,36 @@ limit. These limits are controlled by settings_pack::active_tracker_limit, settings_pack::active_dht_limit and settings_pack::active_lsd_limit respectively. -A client that is not concerned about the separate costs of these actions should -set all 3 of these limits to the same value as settings_pack::active_limit (i.e. -the max limit of any active torrent). +Specifically, announcing to a tracker is typically cheaper than +announcing to the DHT. ``active_dht_limit`` will limit the number of +torrents that are allowed to announce to the DHT. The highest priority ones +will, and the lower priority ones won't. The will still be considered started +though, and any incoming peers will still be accepted. -At a regular interval, torrents are checked if there needs to be any -re-ordering of which torrents are active and which are queued. This interval -can be controlled via settings_pack::auto_manage_interval. +If you do not wish to impose such limits (basically, if you do not wish to have +half-started torrents) make sure to set these limits to -1 (infinite). -For queuing to work, resume data needs to be saved and restored for all -torrents. See save_resume_data(). +prefer seeds +............ -downloading ------------ +In the case where ``active_downloads`` + ``active_seeds`` > ``active_limit``, +there's an ambiguity whether the downloads should be satisfied first or the +seeds. To disambiguate this case, the settings_pack::auto_manage_prefer_seeds +determines whether seeds are preferred or not. -Torrents that are currently being downloaded or incomplete (with bytes still to -download) are queued. The torrents in the front of the queue are started to be -actively downloaded and the rest are ordered with regards to their queue -position. Any newly added torrent is placed at the end of the queue. Once a -torrent is removed or turns into a seed, its queue position is -1 and all -torrents that used to be after it in the queue, decreases their position in -order to fill the gap. +inactive torrents +................. -The queue positions are always in a sequence without any gaps. +Torrents that are not transferring any bytes (downloading or uploading) have a +relatively low cost to be started. It's possible to exempt such torrents from +the download and seed queues by setting settings_pack::dont_count_slow_torrents +to true. -Lower queue position means closer to the front of the queue, and will be -started sooner than torrents with higher queue positions. - -To query a torrent for its position in the queue, or change its position, see: -queue_position(), queue_position_up(), queue_position_down(), -queue_position_top() and queue_position_bottom(). - -seeding -------- - -Auto managed seeding torrents are rotated, so that all of them are allocated a -fair amount of seeding. Torrents with fewer completed *seed cycles* are -prioritized for seeding. A seed cycle is completed when a torrent meets either -the share ratio limit (uploaded bytes / downloaded bytes), the share time ratio -(time seeding / time downloaing) or seed time limit (time seeded). - -The relevant settings to control these limits are -settings_pack::share_ratio_limit, settings_pack::seed_time_ratio_limit and -settings_pack::seed_time_limit. +Since it sometimes may take a few minutes for a newly started torrent to find +peers and be unchoked, or find peers that are interested in requesting data, +torrents are not considered inactive immadiately. There must be an extended +period of no transfers before it is considered inactive and exempt from the +queuing limits. fast resume =========== diff --git a/include/libtorrent/settings_pack.hpp b/include/libtorrent/settings_pack.hpp index 8ae5d074e..3391ce14a 100644 --- a/include/libtorrent/settings_pack.hpp +++ b/include/libtorrent/settings_pack.hpp @@ -978,6 +978,10 @@ namespace libtorrent // ``active_downloads`` and ``active_seeds`` are upper limits on the // number of downloading torrents and seeding torrents respectively. // Setting the value to -1 means unlimited. + // + // ``active_checking`` is the limit of number of checking torrents. + // Note that this limit applies to started non-auto-managed torrents as + // well (as long as they are the the checking_files state). // // For example if there are 10 seeding torrents and 10 downloading // torrents, and ``active_downloads`` is 4 and ``active_seeds`` is 4, @@ -987,8 +991,9 @@ namespace libtorrent // active. Torrents that are not auto managed are not counted against // these limits. // - // ``active_limit`` is a hard limit on the number of active torrents. - // This applies even to slow torrents. + // ``active_limit`` is a hard limit on the number of active (auto + // managed) torrents. This limit also applies to slow torrents. It does + // not apply to checking torrents. // // ``active_dht_limit`` is the max number of torrents to announce to // the DHT. By default this is set to 88, which is no more than one @@ -1019,6 +1024,7 @@ namespace libtorrent // see dynamic-loading-of-torrent-files_. active_downloads, active_seeds, + active_checking, active_dht_limit, active_tracker_limit, active_lsd_limit, diff --git a/include/libtorrent/torrent.hpp b/include/libtorrent/torrent.hpp index cc3ede5a8..281e43e38 100644 --- a/include/libtorrent/torrent.hpp +++ b/include/libtorrent/torrent.hpp @@ -1458,9 +1458,11 @@ namespace libtorrent // ---- - // total time we've been available on this torrent - // does not count when the torrent is stopped or paused - // in seconds + // total time we've been active on this torrent. i.e. either (trying to) + // download or seed. does not count time when the torrent is stopped or + // paused. specified in seconds. This only track time _before_ we started + // the torrent this last time. When the torrent is paused, this counter is + // incremented to include this current session. unsigned int m_active_time:24; // the index to the last tracker that worked @@ -1468,8 +1470,8 @@ namespace libtorrent // ---- - // total time we've been finished with this torrent - // does not count when the torrent is stopped or paused + // total time we've been finished with this torrent. + // does not count when the torrent is stopped or paused. unsigned int m_finished_time:24; // in case the piece picker hasn't been constructed @@ -1512,8 +1514,11 @@ namespace libtorrent // ---- - // total time we've been available as a seed on this torrent - // does not count when the torrent is stopped or paused + // total time we've been available as a seed on this torrent. + // does not count when the torrent is stopped or paused. This value only + // accounts for the time prior to the current start of the torrent. When + // the torrent is paused, this counter is incremented to account for the + // additional seeding time. unsigned int m_seeding_time:24; // ---- diff --git a/include/libtorrent/torrent_handle.hpp b/include/libtorrent/torrent_handle.hpp index e1985bad3..5e3b3ed8c 100644 --- a/include/libtorrent/torrent_handle.hpp +++ b/include/libtorrent/torrent_handle.hpp @@ -545,10 +545,12 @@ namespace libtorrent // requested from it, it is disconnected. This is a graceful shut down of // the torrent in the sense that no downloaded bytes are wasted. // - // torrents that are auto-managed may be automatically resumed again. It - // does not make sense to pause an auto-managed torrent without making it - // not automanaged first. Torrents are auto-managed by default when added - // to the session. For more information, see queuing_. + // .. note:: + // Torrents that are auto-managed may be automatically resumed again. It + // does not make sense to pause an auto-managed torrent without making it + // not automanaged first. Torrents are auto-managed by default when added + // to the session. For more information, see queuing_. + // void pause(int flags = 0) const; void resume() const; diff --git a/src/session_impl.cpp b/src/session_impl.cpp index ae8393955..7bdd281f4 100644 --- a/src/session_impl.cpp +++ b/src/session_impl.cpp @@ -3556,17 +3556,18 @@ retry: torrent* t = *i; TORRENT_ASSERT(t->state() == torrent_status::checking_files); - if (limit <= 0) - { - t->pause(); - } - else - { - t->resume(); - t->start_checking(); - --limit; - } + TORRENT_ASSERT(t->is_auto_managed()); + if (limit <= 0) + { + t->pause(); } + else + { + t->resume(); + t->start_checking(); + --limit; + } + } } void session_impl::auto_manage_torrents(std::vector& list @@ -3629,18 +3630,20 @@ retry: // these counters are set to the number of torrents // of each kind we're allowed to have active - int num_downloaders = settings().get_int(settings_pack::active_downloads); - int num_seeds = settings().get_int(settings_pack::active_seeds); - int checking_limit = 1; + int downloading_limit = settings().get_int(settings_pack::active_downloads); + int seeding_limit = settings().get_int(settings_pack::active_seeds); + int checking_limit = settings().get_int(settings_pack::active_checking); int dht_limit = settings().get_int(settings_pack::active_dht_limit); int tracker_limit = settings().get_int(settings_pack::active_tracker_limit); int lsd_limit = settings().get_int(settings_pack::active_lsd_limit); int hard_limit = settings().get_int(settings_pack::active_limit); - if (num_downloaders == -1) - num_downloaders = (std::numeric_limits::max)(); - if (num_seeds == -1) - num_seeds = (std::numeric_limits::max)(); + if (downloading_limit == -1) + downloading_limit = (std::numeric_limits::max)(); + if (seeding_limit == -1) + seeding_limit = (std::numeric_limits::max)(); + if (checking_limit == -1) + checking_limit = (std::numeric_limits::max)(); if (hard_limit == -1) hard_limit = (std::numeric_limits::max)(); if (dht_limit == -1) @@ -3650,20 +3653,6 @@ retry: if (tracker_limit == -1) tracker_limit = (std::numeric_limits::max)(); - // deduct "force started" torrents from the hard_limit - // we don't have explicit access to the number of force started torrents, - // but we know how many started downloading and seeding torrents we have. - // if we subtract all non-force started torrents from the total, we get - // the number of force started. - hard_limit -= m_stats_counters[counters::num_downloading_torrents] - - downloaders.size(); - hard_limit -= m_stats_counters[counters::num_seeding_torrents] - + m_stats_counters[counters::num_upload_only_torrents] - - seeds.size(); - - // TODO: 3 also deduct force started checking torrents from checking_limit - // also deduct started inactive torrents from hard_limit - // if hard_limit is <= 0, all torrents in these lists should be paused. // The order is not relevant if (hard_limit > 0) @@ -3691,16 +3680,16 @@ retry: if (settings().get_bool(settings_pack::auto_manage_prefer_seeds)) { auto_manage_torrents(seeds, dht_limit, tracker_limit, lsd_limit - , hard_limit, num_seeds); + , hard_limit, seeding_limit); auto_manage_torrents(downloaders, dht_limit, tracker_limit, lsd_limit - , hard_limit, num_downloaders); + , hard_limit, downloading_limit); } else { auto_manage_torrents(downloaders, dht_limit, tracker_limit, lsd_limit - , hard_limit, num_downloaders); + , hard_limit, downloading_limit); auto_manage_torrents(seeds, dht_limit, tracker_limit, lsd_limit - , hard_limit, num_seeds); + , hard_limit, seeding_limit); } } diff --git a/src/settings_pack.cpp b/src/settings_pack.cpp index fa6beb3ec..26ae8a6f8 100644 --- a/src/settings_pack.cpp +++ b/src/settings_pack.cpp @@ -260,6 +260,7 @@ namespace libtorrent SET(peer_tos, 0, &session_impl::update_peer_tos), SET(active_downloads, 3, &session_impl::trigger_auto_manage), SET(active_seeds, 5, &session_impl::trigger_auto_manage), + SET_NOPREV(active_checking, 1, &session_impl::trigger_auto_manage), SET(active_dht_limit, 88, 0), SET(active_tracker_limit, 1600, 0), SET(active_lsd_limit, 60, 0), diff --git a/src/torrent.cpp b/src/torrent.cpp index 9cff64746..311236347 100644 --- a/src/torrent.cpp +++ b/src/torrent.cpp @@ -8082,23 +8082,27 @@ namespace libtorrent bool is_downloading = false; bool is_seeding = false; - if (m_state == torrent_status::checking_files - && (is_auto_managed() || allows_peers())) + if (is_auto_managed() && !has_error()) { - is_checking = true; - } - else if (is_auto_managed() && !has_error() - && (is_paused() - || !is_inactive() - || !settings().get_bool(settings_pack::dont_count_slow_torrents))) - { - // torrents that are started (not paused) and - // inactive are not part of any list. They will not be touched because - // they are inactive - if (is_finished()) - is_seeding = true; - else - is_downloading = true; + if (m_state == torrent_status::checking_files + || m_state == torrent_status::allocating) + { + is_checking = true; + } + else if (m_state == torrent_status::downloading_metadata + || m_state == torrent_status::downloading + || m_state == torrent_status::finished + || m_state == torrent_status::seeding + || m_state == torrent_status::downloading) + { + // torrents that are started (not paused) and + // inactive are not part of any list. They will not be touched because + // they are inactive + if (is_finished()) + is_seeding = true; + else + is_downloading = true; + } } update_list(aux::session_interface::torrent_downloading_auto_managed @@ -8825,20 +8829,24 @@ namespace libtorrent bool is_downloading = false; bool is_seeding = false; - if (m_state == torrent_status::checking_files - && (is_auto_managed() || allows_peers())) + if (is_auto_managed() && !has_error()) { - is_checking = true; - } - else if (is_auto_managed() && !has_error() - && (is_paused() - || !is_inactive() - || !settings().get_bool(settings_pack::dont_count_slow_torrents))) - { - if (is_finished()) - is_seeding = true; - else - is_downloading = true; + if (m_state == torrent_status::checking_files + || m_state == torrent_status::allocating) + { + is_checking = true; + } + else if (m_state == torrent_status::downloading_metadata + || m_state == torrent_status::downloading + || m_state == torrent_status::finished + || m_state == torrent_status::seeding + || m_state == torrent_status::downloading) + { + if (is_finished()) + is_seeding = true; + else + is_downloading = true; + } } TORRENT_ASSERT(m_links[aux::session_interface::torrent_checking_auto_managed].in_list() @@ -9359,9 +9367,9 @@ namespace libtorrent enum flags { seed_ratio_not_met = 0x40000000, - no_seeds = 0x20000000, - recently_started = 0x10000000, - prio_mask = 0x0fffffff + no_seeds = 0x20000000, + recently_started = 0x10000000, + prio_mask = 0x0fffffff }; if (!is_finished()) return 0; @@ -9974,18 +9982,27 @@ namespace libtorrent int torrent::finished_time() const { + // m_finished_time does not account for the current "session", just the + // time before we last started this torrent. To get the current time, we + // need to add the time since we started it return m_finished_time + ((!is_finished() || is_paused()) ? 0 : (m_ses.session_time() - m_became_finished)); } int torrent::active_time() const { + // m_active_time does not account for the current "session", just the + // time before we last started this torrent. To get the current time, we + // need to add the time since we started it return m_active_time + (is_paused() ? 0 : m_ses.session_time() - m_started); } int torrent::seeding_time() const { + // m_seeding_time does not account for the current "session", just the + // time before we last started this torrent. To get the current time, we + // need to add the time since we started it return m_seeding_time + ((!is_seed() || is_paused()) ? 0 : m_ses.session_time() - m_became_seed); }