made get_download_queue() use considerably less memory and scale with larger piece_sizes

ChangeLog

@@ -1,3 +1,5 @@
+	* change in API for block_info in partial_piece_info, instead of
+	  accessing 'peer', call 'peer()'
 	* added support for fully automatic unchoker (no need to specify
 	  number of upload slots). This is on by default
 	* added support for changing socket buffer sizes through

docs/manual.rst

@@ -2562,9 +2562,9 @@ requested. The entry in the vector (``partial_piece_info``) looks like this::
 	{
 		int piece_index;
 		int blocks_in_piece;
-		block_info blocks[256];
 		enum state_t { none, slow, medium, fast };
 		state_t piece_state;
+		block_info* blocks;
 	};
 
 ``piece_index`` is the index of the piece in question. ``blocks_in_piece`` is the
@@ -2585,12 +2585,20 @@ downloaded pieces down. Pieces set to ``none`` can be converted into any of ``fa
 		enum block_state_t
 		{ none, requested, writing, finished };
 
-		tcp::endpoint peer;
+		void set_peer(tcp::endpoint const& ep);
+		tcp::endpoint peer() const;
+
+		unsigned bytes_progress:15;
+		unsigned block_size:15;
 		unsigned state:2;
 		unsigned num_peers:14;
 	};
 
 
+The ``blocks`` field points to an array of ``blocks_in_piece`` elements. This pointer is
+only valid until the next call to ``get_download_queue()`` for any torrent in the same session.
+They all share the storaga for the block arrays in their session object.
+
 The ``block_info`` array contains data for each individual block in the piece. Each block has
 a state (``state``) which is any of:
 
@@ -2603,6 +2611,8 @@ The ``peer`` field is the ip address of the peer this block was downloaded from.
 ``num_peers`` is the number of peers that is currently requesting this block. Typically this
 is 0 or 1, but at the end of the torrent blocks may be requested by more peers in parallel to
 speed things up.
+``bytes_progress`` is the number of bytes that have been received for this block, and
+``block_size`` is the total number of bytes in this block.
 
 get_peer_info()
 ---------------

examples/client_test.cpp

@@ -1360,7 +1360,7 @@ int main(int argc, char* argv[])
 					out << to_string(i->piece_index, 4) << ": [";
 					for (int j = 0; j < i->blocks_in_piece; ++j)
 					{
-						int index = peer_index(i->blocks[j].peer, peers);
+						int index = peer_index(i->blocks[j].peer(), peers);
 						char str[] = "+";
 						if (index >= 0)
 							str[0] = (index < 10)?'0' + index:'A' + index - 10;

include/libtorrent/aux_/session_impl.hpp

@@ -654,6 +654,11 @@ namespace libtorrent
 			size_type m_total_failed_bytes;
 			size_type m_total_redundant_bytes;
 
+			// this vector is used to store the block_info
+			// objects pointed to by partial_piece_info returned
+			// by torrent::get_download_queue.
+			std::vector<block_info> m_block_info_storage;
+
 			// the main working thread
 			boost::scoped_ptr<boost::thread> m_thread;
 		};

include/libtorrent/torrent_handle.hpp

@@ -280,11 +280,43 @@ namespace libtorrent
 		enum block_state_t
 		{ none, requested, writing, finished };
 
-		tcp::endpoint peer;
+	private:
+		union
+		{
+			address_v4::bytes_type v4;
+			address_v6::bytes_type v6;
+		} addr;
+
+		boost::uint16_t port;
+	public:
+
+		void set_peer(tcp::endpoint const& ep)
+		{
+			is_v6_addr = ep.address().is_v6();
+			if (is_v6_addr)
+				addr.v6 = ep.address().to_v6().to_bytes();
+			else
+				addr.v4 = ep.address().to_v4().to_bytes();
+			port = ep.port();
+		}
+
+		tcp::endpoint peer() const
+		{
+			if (is_v6_addr)
+				return tcp::endpoint(address_v6(addr.v6), port);
+			else
+				return tcp::endpoint(address_v4(addr.v4), port);
+		}
+
 		// number of bytes downloaded in this block
-		unsigned bytes_progress:16;
+		unsigned bytes_progress:15;
 		// the total number of bytes in this block
-		unsigned block_size:16;
+		unsigned block_size:15;
+	private:
+		// the type of the addr union
+		unsigned is_v6_addr:1;
+		unsigned unused:1;
+	public:
 		// the state this block is in (see block_state_t)
 		unsigned state:2;
 		// the number of peers that has requested this block
@@ -295,7 +327,6 @@ namespace libtorrent
 
 	struct TORRENT_EXPORT partial_piece_info
 	{
-		enum { max_blocks_per_piece = 512 };
 		int piece_index;
 		int blocks_in_piece;
 		// the number of blocks in the finished state
@@ -304,7 +335,7 @@ namespace libtorrent
 		int writing;
 		// the number of blocks in the requested state
 		int requested;
-		block_info blocks[max_blocks_per_piece];
+		block_info* blocks;
 		enum state_t { none, slow, medium, fast };
 		state_t piece_state;
 	};

src/torrent.cpp

@@ -3422,13 +3422,20 @@ namespace libtorrent
 	void torrent::get_download_queue(std::vector<partial_piece_info>& queue)
 	{
 		queue.clear();
+		std::vector<block_info>& blk = m_ses.m_block_info_storage;
+		blk.clear();
+
 		if (!valid_metadata() || is_seed()) return;
 		piece_picker const& p = picker();
 		std::vector<piece_picker::downloading_piece> const& q
 			= p.get_download_queue();
 
+		const int blocks_per_piece = m_picker->blocks_in_piece(0);
+		blk.resize(q.size() * blocks_per_piece);
+
+		int counter = 0;
 		for (std::vector<piece_picker::downloading_piece>::const_iterator i
-			= q.begin(); i != q.end(); ++i)
+			= q.begin(); i != q.end(); ++i, ++counter)
 		{
 			partial_piece_info pi;
 			pi.piece_state = (partial_piece_info::state_t)i->state;
@@ -3436,9 +3443,9 @@ namespace libtorrent
 			pi.finished = (int)i->finished;
 			pi.writing = (int)i->writing;
 			pi.requested = (int)i->requested;
+			pi.blocks = &blk[counter * blocks_per_piece];
 			int piece_size = int(torrent_file().piece_size(i->index));
-			int num_blocks = (std::min)(pi.blocks_in_piece, int(partial_piece_info::max_blocks_per_piece));
+			for (int j = 0; j < pi.blocks_in_piece; ++j)
-			for (int j = 0; j < num_blocks; ++j)
 			{
 				block_info& bi = pi.blocks[j];
 				bi.state = i->info[j].state;
@@ -3448,7 +3455,7 @@ namespace libtorrent
 					|| bi.state == block_info::finished;
 				if (i->info[j].peer == 0)
 				{
-					bi.peer = tcp::endpoint();
+					bi.set_peer(tcp::endpoint());
 					bi.bytes_progress = complete ? bi.block_size : 0;
 				}
 				else
@@ -3456,7 +3463,7 @@ namespace libtorrent
 					policy::peer* p = static_cast<policy::peer*>(i->info[j].peer);
 					if (p->connection)
 					{
-						bi.peer = p->connection->remote();
+						bi.set_peer(p->connection->remote());
 						if (bi.state == block_info::requested)
 						{
 							boost::optional<piece_block_progress> pbp
@@ -3478,7 +3485,7 @@ namespace libtorrent
 					}
 					else
 					{
-						bi.peer = p->ip();
+						bi.set_peer(p->ip());
 						bi.bytes_progress = complete ? bi.block_size : 0;
 					}
 				}