optimize recalculate auto-managed

include/libtorrent/aux_/session_impl.hpp

@@ -601,13 +601,14 @@ namespace libtorrent
 			libtorrent::utp_socket_manager* utp_socket_manager() { return &m_utp_socket_manager; }
 			void inc_boost_connections() { ++m_boost_connections; }
 
-//		private:
+		private:
 
 			std::vector<torrent*> m_torrent_lists[num_torrent_lists];
 
 			peer_class_pool m_classes;
 
-//		private:
+			// TODO: 2 fix this
+		public:
 
 			void submit_disk_jobs();
 
@@ -1170,7 +1171,7 @@ namespace libtorrent
 
 			// is true if the session is paused
 			bool m_paused;
-			
+
 #ifndef TORRENT_NO_DEPRECATE
 			std::vector<boost::shared_ptr<feed> > m_feeds;
 #endif
@@ -1190,7 +1191,7 @@ namespace libtorrent
 			pthread_t m_network_thread;
 #endif
 		};
-		
+
 #ifndef TORRENT_DISABLE_LOGGING
 		struct tracker_logger : request_callback
 		{

include/libtorrent/aux_/session_interface.hpp

@@ -39,13 +39,12 @@ POSSIBILITY OF SUCH DAMAGE.
 #include "libtorrent/io_service.hpp"
 #include "libtorrent/disk_buffer_holder.hpp"
 
-#ifndef TORRENT_DISABLE_DHT	
+#ifndef TORRENT_DISABLE_DHT
 #include "libtorrent/socket.hpp"
 #endif
 
 #include "libtorrent/socket.hpp" // for tcp::endpoint
 
-
 #include "libtorrent/aux_/disable_warnings_push.hpp"
 
 #include <boost/weak_ptr.hpp>
@@ -298,6 +297,15 @@ namespace libtorrent { namespace aux
 			// torrents that want auto-scrape (only paused auto-managed ones)
 			torrent_want_scrape,
 
+			// auto-managed torrents by state. Only these torrents are considered
+			// when recalculating auto-managed torrents. started auto managed
+			// torrents that are inactive are not part of these lists, because they
+			// are not considered for auto managing (they are left started
+			// unconditionallty)
+			torrent_downloading_auto_managed,
+			torrent_seeding_auto_managed,
+			torrent_checking_auto_managed,
+
 			// all torrents that have resume data to save
 //			torrent_want_save_resume,
 

include/libtorrent/torrent.hpp

@@ -289,7 +289,7 @@ namespace libtorrent
 
 		// This may be called from multiple threads
 		sha1_hash const& info_hash() const { return m_info_hash; }
-	
+
 		bool is_deleted() const { return m_deleted; }
 
 		// starts the announce timer
@@ -644,6 +644,7 @@ namespace libtorrent
 
 		bool want_tick() const;
 		void update_want_tick();
+		void update_state_list();
 
 		bool want_peers() const;
 		bool want_peers_download() const;
@@ -1368,6 +1369,9 @@ namespace libtorrent
 		// efficient to enumerate only torrents belonging to a specific
 		// group. Such as torrents that want peer connections or want
 		// to be ticked etc.
+
+		// TODO: 3 factor out the links (as well as update_list() to a separate
+		// class that torrent can inherit)
 		link m_links[aux::session_interface::num_torrent_lists];
 
 	private:
@@ -1708,10 +1712,12 @@ namespace libtorrent
 		// millionths of completeness)
 		unsigned int m_progress_ppm:20;
 
-		// this is a timestamp of the last time this torrent changed its
-		// m_inactive state. The inactive state is set when transfer rates are
-		// below the active threshold. It's specified in session time unit.
-		boost::int16_t m_last_active_change;
+		// this is true when our effective inactive state is different from our
+		// actual inactive state. Whenever this state changes, there is a
+		// quarantine period until we change the effective state. This is to avoid
+		// flapping. If the state changes back during this period, we cancel the
+		// quarantine
+		bool m_pending_active_change:1;
 
 		// if this is set, accept the save path saved in the resume data, if
 		// present

include/libtorrent/torrent_handle.hpp

@@ -1308,7 +1308,7 @@ namespace libtorrent
 			// large number of torrents at once, they will queue up.
 			checking_resume_data
 		};
-		
+
 		// may be set to an error message describing why the torrent
 		// was paused, in case it was paused by an error. If the torrent
 		// is not paused or if it's paused but not because of an error,

src/session_impl.cpp

@@ -3470,14 +3470,6 @@ retry:
 			t->set_announce_to_trackers(tracker_limit >= 0);
 			t->set_announce_to_lsd(lsd_limit >= 0);
 
-			if (!t->is_paused() && t->is_inactive()
-				&& hard_limit > 0)
-			{
-				// the hard limit takes inactive torrents into account, but the
-				// download and seed limits don't.
-				continue;
-			}
-
 			if (type_limit > 0 && hard_limit > 0)
 			{
 				--hard_limit;
@@ -3509,22 +3501,14 @@ retry:
 
 		if (is_paused()) return;
 
-		// these vectors are filled with auto managed torrents
-
-		// TODO: these vectors could be copied from m_torrent_lists,
-		// if we would maintain them. That way the first pass over
-		// all torrents could be avoided. It would be especially
-		// efficient if most torrents are not auto-managed
-		// whenever we receive a scrape response (or anything
-		// that may change the rank of a torrent) that one torrent
-		// could re-sort itself in a list that's kept sorted at all
-		// times. That way, this pass over all torrents could be
-		// avoided alltogether.
-		std::vector<torrent*> checking;
-		std::vector<torrent*> downloaders;
-		downloaders.reserve(m_torrents.size());
-		std::vector<torrent*> seeds;
-		seeds.reserve(m_torrents.size());
+		// make copies of the lists of torrents that we want to consider for auto
+		// management. We need copies because they will be sorted.
+		std::vector<torrent*> checking
+			= torrent_list(session_interface::torrent_checking_auto_managed);
+		std::vector<torrent*> downloaders
+			= torrent_list(session_interface::torrent_downloading_auto_managed);
+		std::vector<torrent*> seeds
+			= torrent_list(session_interface::torrent_seeding_auto_managed);
 
 		// these counters are set to the number of torrents
 		// of each kind we're allowed to have active
@@ -3549,60 +3533,20 @@ retry:
 		if (tracker_limit == -1)
 			tracker_limit = (std::numeric_limits<int>::max)();
 
-		for (torrent_map::iterator i = m_torrents.begin()
-			, end(m_torrents.end()); i != end; ++i)
-		{
-			torrent* t = i->second.get();
-			TORRENT_ASSERT(t);
+		// TODO: 3 deduct "force started" torrents from the hard_limit
+		// also deduct force started checking torrents from checking_limit
+		// also deduct started inactive torrents from hard_limit
 
-			if (t->state() == torrent_status::checking_files
-				&& !t->has_error()
-				&& (t->is_auto_managed() || t->allows_peers()))
-			{
-				checking.push_back(t);
-				continue;
-			}
+		// TODO: 3 use partial_sort of "type limit" prefix of the list
+		std::sort(checking.begin(), checking.end()
+			, boost::bind(&torrent::sequence_number, _1) < boost::bind(&torrent::sequence_number, _2));
 
-			if (t->is_auto_managed() && !t->has_error())
-			{
-				TORRENT_ASSERT(t->m_resume_data_loaded || !t->valid_metadata());
-				// this torrent is auto managed, add it to
-				// the list (depending on if it's a seed or not)
-				if (t->is_finished())
-					seeds.push_back(t);
-				else
-					downloaders.push_back(t);
-			}
-			else if (!t->is_paused())
-			{
-				if (t->state() == torrent_status::checking_files)
-				{
-					if (checking_limit > 0) --checking_limit;
-					continue;
-				}
-				TORRENT_ASSERT(t->m_resume_data_loaded || !t->valid_metadata());
-				--hard_limit;
-			}
-		}
+		std::sort(downloaders.begin(), downloaders.end()
+			, boost::bind(&torrent::sequence_number, _1) < boost::bind(&torrent::sequence_number, _2));
 
-		bool handled_by_extension = false;
-
-#ifndef TORRENT_DISABLE_EXTENSIONS
-		// TODO: 0 allow extensions to sort torrents for queuing
-#endif
-
-		if (!handled_by_extension)
-		{
-			std::sort(checking.begin(), checking.end()
-				, boost::bind(&torrent::sequence_number, _1) < boost::bind(&torrent::sequence_number, _2));
-
-			std::sort(downloaders.begin(), downloaders.end()
-				, boost::bind(&torrent::sequence_number, _1) < boost::bind(&torrent::sequence_number, _2));
-
-			std::sort(seeds.begin(), seeds.end()
-				, boost::bind(&torrent::seed_rank, _1, boost::ref(m_settings))
-				> boost::bind(&torrent::seed_rank, _2, boost::ref(m_settings)));
-		}
+		std::sort(seeds.begin(), seeds.end()
+			, boost::bind(&torrent::seed_rank, _1, boost::ref(m_settings))
+			> boost::bind(&torrent::seed_rank, _2, boost::ref(m_settings)));
 
 		auto_manage_torrents(checking, checking_limit, dht_limit, tracker_limit, lsd_limit
 			, hard_limit, num_downloaders);

src/torrent.cpp

@@ -270,7 +270,7 @@ namespace libtorrent
 		, m_downloaded(0xffffff)
 		, m_last_scrape((std::numeric_limits<boost::int16_t>::min)())
 		, m_progress_ppm(0)
-		, m_last_active_change(0)
+		, m_pending_active_change(false)
 		, m_use_resume_save_path(p.flags & add_torrent_params::flag_use_resume_save_path)
 	{
 		if (m_pinned)
@@ -346,6 +346,7 @@ namespace libtorrent
 		update_want_peers();
 		update_want_scrape();
 		update_want_tick();
+		update_state_list();
 
 		INVARIANT_CHECK;
 
@@ -4815,7 +4816,7 @@ namespace libtorrent
 			if (alerts().should_post<cache_flushed_alert>())
 				alerts().emplace_alert<cache_flushed_alert>(get_handle());
 		}
-		
+
 		m_storage.reset();
 
 		// TODO: 2 abort lookups this torrent has made via the
@@ -4860,7 +4861,7 @@ namespace libtorrent
 		// the bitfield and that is not currently being super
 		// seeded by any peer
 		TORRENT_ASSERT(m_super_seeding);
-		
+
 		// do a linear search from the first piece
 		int min_availability = 9999;
 		std::vector<int> avail_vec;
@@ -6659,7 +6660,12 @@ namespace libtorrent
 			if (super_seeding_ != -1) super_seeding(super_seeding_);
 
 			int auto_managed_ = rd.dict_find_int_value("auto_managed", -1);
-			if (auto_managed_ != -1) m_auto_managed = auto_managed_;
+			if (auto_managed_ != -1)
+			{
+				m_auto_managed = auto_managed_;
+
+				update_state_list();
+			}
 
 			int sequential_ = rd.dict_find_int_value("sequential_download", -1);
 			if (sequential_ != -1) set_sequential_download(sequential_);
@@ -6675,6 +6681,7 @@ namespace libtorrent
 				update_gauge();
 				update_want_peers();
 				update_want_scrape();
+				update_state_list();
 			}
 			int dht_ = rd.dict_find_int_value("announce_to_dht", -1);
 			if (dht_ != -1) m_announce_to_dht = dht_;
@@ -7872,6 +7879,41 @@ namespace libtorrent
 		update_list(aux::session_interface::torrent_want_tick, want_tick());
 	}
 
+	// this function adjusts which lists this torrent is part of (checking,
+	// seeding or downloading)
+	void torrent::update_state_list()
+	{
+		bool is_checking = false;
+		bool is_downloading = false;
+		bool is_seeding = false;
+
+		if (m_state == torrent_status::checking_files
+			&& (is_auto_managed() || allows_peers()))
+		{
+			is_checking = true;
+		}
+		else if (is_auto_managed() && !has_error()
+			&& (is_paused()
+				|| !is_inactive()
+				|| !settings().get_bool(settings_pack::dont_count_slow_torrents)))
+		{
+			// torrents that are started (not paused) and
+			// inactive are not part of any list. They will not be touched because
+			// they are inactive
+			if (is_finished())
+				is_seeding = true;
+			else
+				is_downloading = true;
+		}
+
+		update_list(aux::session_interface::torrent_downloading_auto_managed
+			, is_downloading);
+		update_list(aux::session_interface::torrent_seeding_auto_managed
+			, is_seeding);
+		update_list(aux::session_interface::torrent_checking_auto_managed
+			, is_checking);
+	}
+
 	// returns true if this torrent is interested in connecting to more peers
 	bool torrent::want_peers() const
 	{
@@ -8045,6 +8087,8 @@ namespace libtorrent
 	// pieces have been downloaded)
 	void torrent::finished()
 	{
+		update_state_list();
+
 		INVARIANT_CHECK;
 
 		TORRENT_ASSERT(is_finished());
@@ -8060,7 +8104,6 @@ namespace libtorrent
 		if (is_seed()) completed();
 
 		send_upload_only();
-
 		state_updated();
 
 		if (m_completed_time == 0)
@@ -8139,6 +8182,7 @@ namespace libtorrent
 #endif
 		send_upload_only();
 		update_want_tick();
+		update_state_list();
 	}
 
 	void torrent::maybe_done_flushing()
@@ -8515,6 +8559,34 @@ namespace libtorrent
 		TORRENT_ASSERT(want_tick() == m_links[aux::session_interface::torrent_want_tick].in_list());
 		TORRENT_ASSERT((!m_allow_peers && m_auto_managed) == m_links[aux::session_interface::torrent_want_scrape].in_list());
 
+		bool is_checking = false;
+		bool is_downloading = false;
+		bool is_seeding = false;
+
+		if (m_state == torrent_status::checking_files
+			&& (is_auto_managed() || allows_peers()))
+		{
+			is_checking = true;
+		}
+		else if (is_auto_managed() && !has_error()
+			&& (is_paused()
+				|| !is_inactive()
+				|| !settings().get_bool(settings_pack::dont_count_slow_torrents)))
+		{
+			if (is_finished())
+				is_seeding = true;
+			else
+				is_downloading = true;
+		}
+
+		TORRENT_ASSERT(m_links[aux::session_interface::torrent_checking_auto_managed].in_list()
+			== is_checking);
+		TORRENT_ASSERT(m_links[aux::session_interface::torrent_downloading_auto_managed].in_list()
+			== is_downloading);
+		TORRENT_ASSERT(m_links[aux::session_interface::torrent_seeding_auto_managed].in_list()
+			== is_seeding);
+
+
 		TORRENT_ASSERT(is_single_thread());
 		// this fires during disconnecting peers
 //		if (is_paused()) TORRENT_ASSERT(num_peers() == 0 || m_graceful_pause_mode);
@@ -8838,6 +8910,8 @@ namespace libtorrent
 
 		update_gauge();
 		state_updated();
+		update_want_peers();
+		update_state_list();
 
 		// if we haven't downloaded the metadata from m_url, try again
 		if (!m_url.empty() && !m_torrent_file->is_valid())
@@ -8891,6 +8965,7 @@ namespace libtorrent
 #endif
 
 		state_updated();
+		update_state_list();
 	}
 
 	void torrent::auto_managed(bool a)
@@ -8903,6 +8978,7 @@ namespace libtorrent
 		m_auto_managed = a;
 		update_gauge();
 		update_want_scrape();
+		update_state_list();
 
 		state_updated();
 
@@ -9361,6 +9437,7 @@ namespace libtorrent
 
 		update_gauge();
 		update_want_scrape();
+		update_state_list();
 
 		if (!b)
 		{
@@ -9804,14 +9881,22 @@ namespace libtorrent
 		// filter in order to avoid flapping (auto_manage_startup).
 		bool is_inactive = is_inactive_internal();
 
-		if (is_inactive != m_inactive
-			&& settings().get_bool(settings_pack::dont_count_slow_torrents))
+		if (settings().get_bool(settings_pack::dont_count_slow_torrents))
 		{
-			m_last_active_change = m_ses.session_time();
-			int delay = settings().get_int(settings_pack::auto_manage_startup);
-			m_inactivity_timer.expires_from_now(seconds(delay));
-			m_inactivity_timer.async_wait(boost::bind(&torrent::on_inactivity_tick
-				, shared_from_this(), _1));
+			if (is_inactive != m_inactive
+				&& !m_pending_active_change)
+			{
+				int delay = settings().get_int(settings_pack::auto_manage_startup);
+				m_inactivity_timer.expires_from_now(seconds(delay));
+				m_inactivity_timer.async_wait(boost::bind(&torrent::on_inactivity_tick
+					, shared_from_this(), _1));
+				m_pending_active_change = true;
+			}
+			else if (is_inactive == m_inactive
+				&& m_pending_active_change)
+			{
+				m_inactivity_timer.cancel();
+			}
 		}
 
 		update_want_tick();
@@ -9829,17 +9914,20 @@ namespace libtorrent
 
 	void torrent::on_inactivity_tick(error_code const& ec)
 	{
+		m_pending_active_change = false;
+
 		if (ec) return;
 
 		int now = m_ses.session_time();
 		int delay = settings().get_int(settings_pack::auto_manage_startup);
-		if (now - m_last_active_change < delay) return;
 
 		bool is_inactive = is_inactive_internal();
 		if (is_inactive == m_inactive) return;
 
 		m_inactive = is_inactive;
 
+		update_state_list();
+
 		if (settings().get_bool(settings_pack::dont_count_slow_torrents))
 			m_ses.trigger_auto_manage();
 	}
@@ -11240,7 +11328,7 @@ namespace libtorrent
 			}
 		}
 	}
-	
+
 	void torrent::new_external_ip()
 	{
 		if (m_peer_list) m_peer_list->clear_peer_prio();
@@ -11288,6 +11376,7 @@ namespace libtorrent
 #endif
 
 		update_want_peers();
+		update_state_list();
 		update_gauge();
 
 		state_updated();
@@ -11319,7 +11408,7 @@ namespace libtorrent
 
 	void torrent::state_updated()
 	{
-		// if this fails, this function is probably called 
+		// if this fails, this function is probably called
 		// from within the torrent constructor, which it
 		// shouldn't be. Whichever function ends up calling
 		// this should probably be moved to torrent::start()

src/torrent_info.cpp

@@ -880,8 +880,9 @@ namespace libtorrent
 		{
 #ifndef BOOST_NO_EXCEPTIONS
 			throw invalid_torrent_file(ec);
-#endif
+#else
 			return;
+#endif
 		}
 #ifndef BOOST_NO_EXCEPTIONS
 		if (!parse_torrent_file(e, ec, 0))
@@ -911,8 +912,9 @@ namespace libtorrent
 		{
 #ifndef BOOST_NO_EXCEPTIONS
 			throw invalid_torrent_file(ec);
-#endif
+#else
 			return;
+#endif
 		}
 #ifndef BOOST_NO_EXCEPTIONS
 		if (!parse_torrent_file(e, ec, 0))