merged RC_1_1 into master

ChangeLog

@@ -82,6 +82,7 @@
 	* resume data no longer has timestamps of files
 	* require C++11 to build libtorrent
 
+	* fix pad-file scalability issue
 	* made coalesce_reads/coalesce_writes settings take effect on linux and windows
 	* restore support for incoming connections over SOCKS5 (disabled by default)
 	* use unique peer_ids per connection

include/libtorrent/piece_picker.hpp

@@ -42,14 +42,7 @@ POSSIBILITY OF SUCH DAMAGE.
 #include <utility>
 #include <cstdint>
 #include <tuple>
-
-#ifdef TORRENT_DEBUG_REFCOUNTS
 #include <set>
-#endif
-
-#if TORRENT_USE_ASSERTS
-#include <set>
-#endif
 
 #include "libtorrent/peer_id.hpp"
 #include "libtorrent/assert.hpp"
@@ -327,6 +320,8 @@ namespace libtorrent {
 		void mark_as_canceled(piece_block block, torrent_peer* peer);
 		void mark_as_finished(piece_block block, torrent_peer* peer);
 
+		void mark_as_pad(piece_block block);
+
 		// prevent blocks from being picked from this piece.
 		// to unlock the piece, call restore_piece() on it
 		void lock_piece(piece_index_t piece);
@@ -711,6 +706,11 @@ namespace libtorrent {
 		// TODO: should this be allocated lazily?
 		mutable aux::vector<piece_pos, piece_index_t> m_piece_map;
 
+		// this maps pieces to a range of blocks that are pad files and should not
+		// be picked
+		// TOOD: this could be a much more efficient data structure
+		std::set<piece_block> m_pad_blocks;
+
 		// the number of seeds. These are not added to
 		// the availability counters of the pieces
 		int m_seeds = 0;

src/piece_picker.cpp

@@ -217,10 +217,21 @@ namespace libtorrent {
 		TORRENT_ASSERT(int(ret.info_idx) * m_blocks_per_piece
 			+ m_blocks_per_piece <= int(m_block_info.size()));
 
+		int block_idx = 0;
 		for (auto& info : mutable_blocks_for_piece(ret))
 		{
 			info.num_peers = 0;
 			info.state = block_info::state_none;
+			if (m_pad_blocks.count(piece_block(piece, block_idx)))
+			{
+				info.state = block_info::state_finished;
+				++ret.finished;
+			}
+			else
+			{
+				info.state = block_info::state_none;
+			}
+			++block_idx;
 			info.peer = nullptr;
 #if TORRENT_USE_ASSERTS
 			info.piece_index = piece;
@@ -229,6 +240,10 @@ namespace libtorrent {
 		}
 		downloading_iter = m_downloads[download_state].insert(downloading_iter, ret);
 
+		// in case every block was a pad block, we need to make sure the piece
+		// structure is correctly categorised
+		downloading_iter = update_piece_state(downloading_iter);
+
 #if TORRENT_USE_INVARIANT_CHECKS
 		check_piece_state();
 #endif
@@ -2977,6 +2992,9 @@ get_out:
 			auto const binfo = mutable_blocks_for_piece(*dp);
 			block_info& info = binfo[block.block_index];
 			TORRENT_ASSERT(info.piece_index == block.piece_index);
+			if (info.state == block_info::state_finished)
+				return false;
+
 			info.state = block_info::state_requested;
 			info.peer = peer;
 			info.num_peers = 1;
@@ -3122,6 +3140,11 @@ get_out:
 			TORRENT_ASSERT(&info >= &m_block_info[0]);
 			TORRENT_ASSERT(&info < &m_block_info[0] + m_block_info.size());
 			TORRENT_ASSERT(info.piece_index == block.piece_index);
+
+			TORRENT_ASSERT(info.state == block_info::state_none);
+			if (info.state == block_info::state_finished)
+				return false;
+
 			info.state = block_info::state_writing;
 			info.peer = peer;
 			info.num_peers = 0;
@@ -3364,6 +3387,8 @@ get_out:
 			TORRENT_ASSERT(&info >= &m_block_info[0]);
 			TORRENT_ASSERT(&info < &m_block_info[0] + m_block_info.size());
 			TORRENT_ASSERT(info.piece_index == block.piece_index);
+			if (info.state == block_info::state_finished)
+				return;
 			info.peer = peer;
 			TORRENT_ASSERT(info.state == block_info::state_none);
 			TORRENT_ASSERT(info.num_peers == 0);
@@ -3422,6 +3447,22 @@ get_out:
 #endif
 	}
 
+	void piece_picker::mark_as_pad(piece_block block)
+	{
+		m_pad_blocks.insert(block);
+		// if we mark and entire piece as a pad file, we need to also
+		// consder that piece as "had" and increment some counters
+		typedef std::set<piece_block>::iterator iter;
+		iter begin = m_pad_blocks.lower_bound(piece_block(block.piece_index, 0));
+		int const blocks = blocks_in_piece(block.piece_index);
+		iter end = m_pad_blocks.upper_bound(piece_block(block.piece_index, blocks));
+		if (std::distance(begin, end) == blocks)
+		{
+			// the entire piece is a pad file
+			we_have(block.piece_index);
+		}
+	}
+
 	void piece_picker::get_downloaders(std::vector<torrent_peer*>& d
 		, piece_index_t const index) const
 	{

src/torrent.cpp

@@ -1803,7 +1803,7 @@ namespace libtorrent {
 				for (; pr.length >= block; pr.length -= block, ++pb.block_index)
 				{
 					if (pb.block_index == blocks_per_piece) { pb.block_index = 0; ++pb.piece_index; }
-					m_picker->mark_as_finished(pb, nullptr);
+					m_picker->mark_as_pad(pb);
 				}
 				// ugly edge case where padfiles are not used they way they're
 				// supposed to be. i.e. added back-to back or at the end

test/test_piece_picker.cpp

@@ -1993,5 +1993,89 @@ TORRENT_TEST(download_filtered_piece)
 	TEST_EQUAL(test_pick(p, piece_picker::rarest_first | piece_picker::prioritize_partials), piece_index_t(0));
 }
 
+TORRENT_TEST(mark_as_pad)
+{
+	auto p = setup_picker("1111111", "       ", "4444444", "");
+	piece_block const bl(piece_index_t{2}, 0);
+	p->mark_as_pad(bl);
+
+	bool ret = p->mark_as_downloading({piece_index_t{2}, 1}, tmp_peer);
+	TEST_EQUAL(ret, true);
+
+	auto dl = p->get_download_queue();
+
+	TEST_EQUAL(dl.size(), 1);
+	TEST_EQUAL(dl[0].finished, 1);
+	TEST_EQUAL(dl[0].writing, 0);
+	TEST_EQUAL(dl[0].requested, 1);
+	TEST_EQUAL(dl[0].index, piece_index_t{2});
+
+	auto blocks = p->blocks_for_piece(dl[0]);
+	TEST_EQUAL(blocks[0].state, piece_picker::block_info::state_finished);
+	TEST_EQUAL(blocks[1].state, piece_picker::block_info::state_requested);
+	TEST_EQUAL(blocks[2].state, piece_picker::block_info::state_none);
+	TEST_EQUAL(blocks[3].state, piece_picker::block_info::state_none);
+}
+
+TORRENT_TEST(mark_as_pad_downloading)
+{
+	auto p = setup_picker("1111111", "       ", "4444444", "");
+	piece_block const bl(piece_index_t{2}, 0);
+	p->mark_as_pad(bl);
+
+	bool ret = p->mark_as_downloading({piece_index_t{2}, 0}, tmp_peer);
+	TEST_EQUAL(ret, false);
+
+	auto dl = p->get_download_queue();
+
+	TEST_EQUAL(dl.size(), 1);
+	TEST_EQUAL(dl[0].finished, 1);
+	TEST_EQUAL(dl[0].writing, 0);
+	TEST_EQUAL(dl[0].requested, 0);
+	TEST_EQUAL(dl[0].index, piece_index_t{2});
+
+	auto blocks = p->blocks_for_piece(dl[0]);
+	TEST_EQUAL(blocks[0].state, piece_picker::block_info::state_finished);
+	TEST_EQUAL(blocks[1].state, piece_picker::block_info::state_none);
+	TEST_EQUAL(blocks[2].state, piece_picker::block_info::state_none);
+	TEST_EQUAL(blocks[3].state, piece_picker::block_info::state_none);
+}
+
+TORRENT_TEST(mark_as_pad_seeding)
+{
+	auto p = setup_picker("1", " ", "4", "");
+	p->mark_as_pad({piece_index_t{0}, 0});
+	p->mark_as_pad({piece_index_t{0}, 1});
+	p->mark_as_pad({piece_index_t{0}, 2});
+
+	TEST_CHECK(!p->is_seeding());
+
+	p->mark_as_finished({piece_index_t{0}, 3}, tmp_peer);
+
+	TEST_CHECK(!p->is_seeding());
+	p->piece_passed(piece_index_t{0});
+	TEST_CHECK(p->is_seeding());
+}
+
+TORRENT_TEST(mark_as_pad_whole_piece_seeding)
+{
+	auto p = setup_picker("11", "  ", "44", "");
+	p->mark_as_pad({piece_index_t{0}, 0});
+	p->mark_as_pad({piece_index_t{0}, 1});
+	p->mark_as_pad({piece_index_t{0}, 2});
+	p->mark_as_pad({piece_index_t{0}, 3});
+
+	TEST_CHECK(!p->is_seeding());
+
+	p->mark_as_finished({piece_index_t{1}, 0}, NULL);
+	p->mark_as_finished({piece_index_t{1}, 1}, NULL);
+	p->mark_as_finished({piece_index_t{1}, 2}, NULL);
+	p->mark_as_finished({piece_index_t{1}, 3}, NULL);
+
+	TEST_CHECK(!p->is_seeding());
+	p->piece_passed(piece_index_t{1});
+	TEST_CHECK(p->is_seeding());
+}
+
 //TODO: 2 test picking with partial pieces and other peers present so that both
 // backup_pieces and backup_pieces2 are used