end-game mode optimizations

ChangeLog

@@ -81,6 +81,7 @@
 	  incoming connection
 	* added more detailed instrumentation of the disk I/O thread
 
+	* end-game mode optimizations
 	* fixed bug in udp_socket causing it to issue two simultaneous async. read operations
 	* fixed mingw build
 	* fixed minor bug in metadata block requester (for magnet links)

docs/manual.rst

@@ -4304,6 +4304,7 @@ session_settings
 		bool no_connect_privileged_ports;
 		int alert_queue_size;
 		int max_metadata_size;
+		int max_duplicate_block_requests;
 	};
 
 ``version`` is automatically set to the libtorrent version you're using
@@ -5137,6 +5138,12 @@ defaults to 1000.
 ``max_metadata_size`` is the maximum allowed size (in bytes) to be received
 by the metadata extension, i.e. magnet links. It defaults to 1 MiB.
 
+``max_duplicate_block_requests`` is the max number of simultaneous outstanding
+requests ot have for a single block. It defaults to 7. It only applies in end
+game mode where a single block may be requested from multiple peers. Setting
+this too high may cause excessive redundant data being downloaded, setting it
+too low may slow down completion towards the end of a torrent download.
+
 pe_settings
 ===========
 

include/libtorrent/aux_/session_impl.hpp

@@ -874,7 +874,7 @@ namespace libtorrent
 			// to the end-game mode
 			int m_end_game_piece_picks;
 			int m_strict_end_game_piece_picks;
-			int m_valid_strict_end_game_piece_picks;
+			int m_piece_picker_blocks;
 #endif
 
 			// each second tick the timer takes a little

include/libtorrent/piece_picker.hpp

@@ -49,7 +49,6 @@ POSSIBILITY OF SUCH DAMAGE.
 #endif
 
 #include "libtorrent/peer_id.hpp"
-#include "libtorrent/session_settings.hpp"
 #include "libtorrent/config.hpp"
 #include "libtorrent/assert.hpp"
 #include "libtorrent/time.hpp"
@@ -147,9 +146,8 @@ namespace libtorrent
 
 		struct downloading_piece
 		{
-			downloading_piece(): last_request(min_time()), finished(0), writing(0), requested(0) {}
+			downloading_piece(): finished(0), writing(0), requested(0) {}
 			piece_state_t state;
-			ptime last_request;
 
 			// the index of the piece
 			int index;
@@ -290,7 +288,6 @@ namespace libtorrent
 		void mark_as_finished(piece_block block, void* peer);
 		void write_failed(piece_block block);
 		int num_peers(piece_block block) const;
-		ptime last_request(int piece) const;
 
 		// returns information about the given piece
 		void piece_info(int index, piece_picker::downloading_piece& st) const;

include/libtorrent/session_settings.hpp

@@ -263,6 +263,7 @@ namespace libtorrent
 			, no_connect_privileged_ports(true)
 			, alert_queue_size(1000)
 			, max_metadata_size(1024*1024)
+			, max_duplicate_block_requests(7)
 		{}
 
 		// libtorrent version. Used for forward binary compatibility
@@ -1046,6 +1047,12 @@ namespace libtorrent
 		// the max allowed size for metadata received by the
 		// ut_metadata extension (i.e. magnet links)
 		int max_metadata_size;
+
+		// the max number of requests to send for a block. This
+		// is relevant in end-game mode. If a block has been requested
+		// this many times, we won't request it again from any other
+		// peer until at least one of the requests have timed out
+		int max_duplicate_block_requests;
 	};
 
 #ifndef TORRENT_DISABLE_DHT

include/libtorrent/storage.hpp

@@ -69,6 +69,7 @@ namespace libtorrent
 	struct file_pool;
 	struct disk_io_job;
 	struct disk_buffer_pool;
+	struct session_settings;
 
 	TORRENT_EXPORT std::vector<std::pair<size_type, std::time_t> > get_filesizes(
 		file_storage const& t

src/peer_connection.cpp

@@ -3851,7 +3851,7 @@ namespace libtorrent
 			&& m_interesting
 			&& m_download_queue.empty()
 			&& m_request_queue.empty()
-			&& total_seconds(now - m_last_request) > 5)
+			&& total_seconds(now - m_last_request) >= 3)
 		{
 			// this happens when we're in strict end-game
 			// mode and the peer could not request any blocks

src/piece_picker.cpp

@@ -2019,7 +2019,6 @@ namespace libtorrent
 			info.peer = peer;
 			info.num_peers = 1;
 			++dp.requested;
-			dp.last_request = time_now();
 		}
 		else
 		{
@@ -2044,7 +2043,6 @@ namespace libtorrent
 			}
 			++info.num_peers;
 			if (i->state == none) i->state = state;
-			i->last_request = time_now();
 		}
 		return true;
 	}
@@ -2067,23 +2065,6 @@ namespace libtorrent
 		return info.num_peers;
 	}
 
-	ptime piece_picker::last_request(int piece) const
-	{
-		TORRENT_ASSERT(piece >= 0);
-		TORRENT_ASSERT(piece < (int)m_piece_map.size());
-
-		piece_pos const& p = m_piece_map[piece];
-		if (!p.downloading) return min_time();
-
-		std::vector<downloading_piece>::const_iterator i
-			= std::find_if(m_downloads.begin(), m_downloads.end(), has_index(piece));
-		TORRENT_ASSERT(i != m_downloads.end());
-		// just to play it safe
-		if (i == m_downloads.end()) return min_time();
-
-		return i->last_request;
-	}
-
 	void piece_picker::get_availability(std::vector<int>& avail) const
 	{
 		TORRENT_ASSERT(m_seeds >= 0);

src/policy.cpp

@@ -227,16 +227,29 @@ namespace libtorrent
 		c.peer_log("*** PIECE_PICKER [ prefer_whole: %d picked: %d ]"
 			, prefer_whole_pieces, int(interesting_pieces.size()));
 #endif
+		aux::session_impl& ses = t.session();
+
 		std::vector<pending_block> const& dq = c.download_queue();
 		std::vector<pending_block> const& rq = c.request_queue();
 		for (std::vector<piece_block>::iterator i = interesting_pieces.begin();
 			i != interesting_pieces.end(); ++i)
 		{
+#ifdef TORRENT_STATS
+			++ses.m_piece_picker_blocks;
+#endif
+
 			if (prefer_whole_pieces == 0 && num_requests <= 0) break;
 
-			if (p.is_requested(*i))
+			int num_block_requests = p.num_peers(*i);
+			if (num_block_requests > 0)
 			{
 				if (num_requests <= 0) break;
+
+				// if this piece already has the max number of requests to it,
+				// no need to consider it, since we won't send another request anyway
+				if (num_block_requests >= ses.m_settings.max_duplicate_block_requests)
+					continue;
+
 				// don't request pieces we already have in our request queue
 				if (std::find_if(dq.begin(), dq.end(), has_block(*i)) != dq.end()
 					|| std::find_if(rq.begin(), rq.end(), has_block(*i)) != rq.end())
@@ -301,7 +314,6 @@ namespace libtorrent
 		}
 
 #ifdef TORRENT_STATS
-		aux::session_impl& ses = t.session();
 		++ses.m_end_game_piece_picks;
 #endif
 
@@ -335,18 +347,6 @@ namespace libtorrent
 		TORRENT_ASSERT(p.is_requested(*i));
 		TORRENT_ASSERT(p.num_peers(*i) > 0);
 
-		ptime last_request = p.last_request(i->piece_index);
-		ptime now = time_now();
-
-		// don't re-request from a piece more often than once every 5 seconds
-		// TODO: make configurable
-		if (now - last_request < seconds(5))
-			return;
-
-#ifdef TORRENT_STATS
-		++ses.m_valid_strict_end_game_piece_picks;
-#endif
-
 		c.add_request(*i, peer_connection::req_busy);
 	}
 

src/session_impl.cpp

@@ -358,6 +358,7 @@ namespace aux {
 		TORRENT_SETTING(boolean, no_connect_privileged_ports)
 		TORRENT_SETTING(integer, alert_queue_size)
 		TORRENT_SETTING(integer, max_metadata_size)
+		TORRENT_SETTING(integer, max_duplicate_block_requests)
 	};
 
 #undef TORRENT_SETTING
@@ -806,7 +807,7 @@ namespace aux {
 			":outstanding writing blocks"
 			":end game piece picks"
 			":strict end game piece picks"
-			":valid strict end game piece picks"
+			":piece picker blocks"
 			":% failed payload bytes"
 			":% wasted payload bytes"
 			":% protocol bytes"
@@ -818,7 +819,7 @@ namespace aux {
 		m_connreset_peers = 0;
 		m_end_game_piece_picks = 0;
 		m_strict_end_game_piece_picks = 0;
-		m_valid_strict_end_game_piece_picks = 0;
+		m_piece_picker_blocks = 0;
 #endif
 #ifdef TORRENT_DISK_STATS
 		m_buffer_usage_logger.open("buffer_stats.log", std::ios::trunc);
@@ -2683,7 +2684,7 @@ namespace aux {
 			<< outstanding_write_blocks << "\t"
 			<< m_end_game_piece_picks << "\t"
 			<< m_strict_end_game_piece_picks << "\t"
-			<< m_valid_strict_end_game_piece_picks << "\t"
+			<< m_piece_picker_blocks << "\t"
 			<< (float(m_total_failed_bytes) * 100.f / m_stat.total_payload_download()) << "\t"
 			<< (float(m_total_redundant_bytes)	* 100.f / m_stat.total_payload_download()) << "\t"
 			<< (float(m_stat.total_protocol_download()) * 100.f / m_stat.total_download()) << "\t"
@@ -2694,7 +2695,7 @@ namespace aux {
 		m_connreset_peers = 0;
 		m_end_game_piece_picks = 0;
 		m_strict_end_game_piece_picks = 0;
-		m_valid_strict_end_game_piece_picks = 0;
+		m_piece_picker_blocks = 0;
 #endif
 
 		// --------------------------------------------------------------