diff --git a/ChangeLog b/ChangeLog index 1526e2c7b..6ede65b45 100644 --- a/ChangeLog +++ b/ChangeLog @@ -82,6 +82,7 @@ * resume data no longer has timestamps of files * require C++11 to build libtorrent + * fix pad-file scalability issue * made coalesce_reads/coalesce_writes settings take effect on linux and windows * restore support for incoming connections over SOCKS5 (disabled by default) * use unique peer_ids per connection diff --git a/include/libtorrent/piece_picker.hpp b/include/libtorrent/piece_picker.hpp index 06686d355..cfe74b8fc 100644 --- a/include/libtorrent/piece_picker.hpp +++ b/include/libtorrent/piece_picker.hpp @@ -42,14 +42,7 @@ POSSIBILITY OF SUCH DAMAGE. #include #include #include - -#ifdef TORRENT_DEBUG_REFCOUNTS #include -#endif - -#if TORRENT_USE_ASSERTS -#include -#endif #include "libtorrent/peer_id.hpp" #include "libtorrent/assert.hpp" @@ -327,6 +320,8 @@ namespace libtorrent { void mark_as_canceled(piece_block block, torrent_peer* peer); void mark_as_finished(piece_block block, torrent_peer* peer); + void mark_as_pad(piece_block block); + // prevent blocks from being picked from this piece. // to unlock the piece, call restore_piece() on it void lock_piece(piece_index_t piece); @@ -711,6 +706,11 @@ namespace libtorrent { // TODO: should this be allocated lazily? mutable aux::vector m_piece_map; + // this maps pieces to a range of blocks that are pad files and should not + // be picked + // TOOD: this could be a much more efficient data structure + std::set m_pad_blocks; + // the number of seeds. These are not added to // the availability counters of the pieces int m_seeds = 0; diff --git a/src/piece_picker.cpp b/src/piece_picker.cpp index d8f47dcaa..84441d5ea 100644 --- a/src/piece_picker.cpp +++ b/src/piece_picker.cpp @@ -217,10 +217,21 @@ namespace libtorrent { TORRENT_ASSERT(int(ret.info_idx) * m_blocks_per_piece + m_blocks_per_piece <= int(m_block_info.size())); + int block_idx = 0; for (auto& info : mutable_blocks_for_piece(ret)) { info.num_peers = 0; info.state = block_info::state_none; + if (m_pad_blocks.count(piece_block(piece, block_idx))) + { + info.state = block_info::state_finished; + ++ret.finished; + } + else + { + info.state = block_info::state_none; + } + ++block_idx; info.peer = nullptr; #if TORRENT_USE_ASSERTS info.piece_index = piece; @@ -229,6 +240,10 @@ namespace libtorrent { } downloading_iter = m_downloads[download_state].insert(downloading_iter, ret); + // in case every block was a pad block, we need to make sure the piece + // structure is correctly categorised + downloading_iter = update_piece_state(downloading_iter); + #if TORRENT_USE_INVARIANT_CHECKS check_piece_state(); #endif @@ -2977,6 +2992,9 @@ get_out: auto const binfo = mutable_blocks_for_piece(*dp); block_info& info = binfo[block.block_index]; TORRENT_ASSERT(info.piece_index == block.piece_index); + if (info.state == block_info::state_finished) + return false; + info.state = block_info::state_requested; info.peer = peer; info.num_peers = 1; @@ -3122,6 +3140,11 @@ get_out: TORRENT_ASSERT(&info >= &m_block_info[0]); TORRENT_ASSERT(&info < &m_block_info[0] + m_block_info.size()); TORRENT_ASSERT(info.piece_index == block.piece_index); + + TORRENT_ASSERT(info.state == block_info::state_none); + if (info.state == block_info::state_finished) + return false; + info.state = block_info::state_writing; info.peer = peer; info.num_peers = 0; @@ -3364,6 +3387,8 @@ get_out: TORRENT_ASSERT(&info >= &m_block_info[0]); TORRENT_ASSERT(&info < &m_block_info[0] + m_block_info.size()); TORRENT_ASSERT(info.piece_index == block.piece_index); + if (info.state == block_info::state_finished) + return; info.peer = peer; TORRENT_ASSERT(info.state == block_info::state_none); TORRENT_ASSERT(info.num_peers == 0); @@ -3422,6 +3447,22 @@ get_out: #endif } + void piece_picker::mark_as_pad(piece_block block) + { + m_pad_blocks.insert(block); + // if we mark and entire piece as a pad file, we need to also + // consder that piece as "had" and increment some counters + typedef std::set::iterator iter; + iter begin = m_pad_blocks.lower_bound(piece_block(block.piece_index, 0)); + int const blocks = blocks_in_piece(block.piece_index); + iter end = m_pad_blocks.upper_bound(piece_block(block.piece_index, blocks)); + if (std::distance(begin, end) == blocks) + { + // the entire piece is a pad file + we_have(block.piece_index); + } + } + void piece_picker::get_downloaders(std::vector& d , piece_index_t const index) const { diff --git a/src/torrent.cpp b/src/torrent.cpp index 05b51865d..1e57d7e47 100644 --- a/src/torrent.cpp +++ b/src/torrent.cpp @@ -1803,7 +1803,7 @@ namespace libtorrent { for (; pr.length >= block; pr.length -= block, ++pb.block_index) { if (pb.block_index == blocks_per_piece) { pb.block_index = 0; ++pb.piece_index; } - m_picker->mark_as_finished(pb, nullptr); + m_picker->mark_as_pad(pb); } // ugly edge case where padfiles are not used they way they're // supposed to be. i.e. added back-to back or at the end diff --git a/test/test_piece_picker.cpp b/test/test_piece_picker.cpp index 6fa384992..df01b3d4d 100644 --- a/test/test_piece_picker.cpp +++ b/test/test_piece_picker.cpp @@ -1993,5 +1993,89 @@ TORRENT_TEST(download_filtered_piece) TEST_EQUAL(test_pick(p, piece_picker::rarest_first | piece_picker::prioritize_partials), piece_index_t(0)); } +TORRENT_TEST(mark_as_pad) +{ + auto p = setup_picker("1111111", " ", "4444444", ""); + piece_block const bl(piece_index_t{2}, 0); + p->mark_as_pad(bl); + + bool ret = p->mark_as_downloading({piece_index_t{2}, 1}, tmp_peer); + TEST_EQUAL(ret, true); + + auto dl = p->get_download_queue(); + + TEST_EQUAL(dl.size(), 1); + TEST_EQUAL(dl[0].finished, 1); + TEST_EQUAL(dl[0].writing, 0); + TEST_EQUAL(dl[0].requested, 1); + TEST_EQUAL(dl[0].index, piece_index_t{2}); + + auto blocks = p->blocks_for_piece(dl[0]); + TEST_EQUAL(blocks[0].state, piece_picker::block_info::state_finished); + TEST_EQUAL(blocks[1].state, piece_picker::block_info::state_requested); + TEST_EQUAL(blocks[2].state, piece_picker::block_info::state_none); + TEST_EQUAL(blocks[3].state, piece_picker::block_info::state_none); +} + +TORRENT_TEST(mark_as_pad_downloading) +{ + auto p = setup_picker("1111111", " ", "4444444", ""); + piece_block const bl(piece_index_t{2}, 0); + p->mark_as_pad(bl); + + bool ret = p->mark_as_downloading({piece_index_t{2}, 0}, tmp_peer); + TEST_EQUAL(ret, false); + + auto dl = p->get_download_queue(); + + TEST_EQUAL(dl.size(), 1); + TEST_EQUAL(dl[0].finished, 1); + TEST_EQUAL(dl[0].writing, 0); + TEST_EQUAL(dl[0].requested, 0); + TEST_EQUAL(dl[0].index, piece_index_t{2}); + + auto blocks = p->blocks_for_piece(dl[0]); + TEST_EQUAL(blocks[0].state, piece_picker::block_info::state_finished); + TEST_EQUAL(blocks[1].state, piece_picker::block_info::state_none); + TEST_EQUAL(blocks[2].state, piece_picker::block_info::state_none); + TEST_EQUAL(blocks[3].state, piece_picker::block_info::state_none); +} + +TORRENT_TEST(mark_as_pad_seeding) +{ + auto p = setup_picker("1", " ", "4", ""); + p->mark_as_pad({piece_index_t{0}, 0}); + p->mark_as_pad({piece_index_t{0}, 1}); + p->mark_as_pad({piece_index_t{0}, 2}); + + TEST_CHECK(!p->is_seeding()); + + p->mark_as_finished({piece_index_t{0}, 3}, tmp_peer); + + TEST_CHECK(!p->is_seeding()); + p->piece_passed(piece_index_t{0}); + TEST_CHECK(p->is_seeding()); +} + +TORRENT_TEST(mark_as_pad_whole_piece_seeding) +{ + auto p = setup_picker("11", " ", "44", ""); + p->mark_as_pad({piece_index_t{0}, 0}); + p->mark_as_pad({piece_index_t{0}, 1}); + p->mark_as_pad({piece_index_t{0}, 2}); + p->mark_as_pad({piece_index_t{0}, 3}); + + TEST_CHECK(!p->is_seeding()); + + p->mark_as_finished({piece_index_t{1}, 0}, NULL); + p->mark_as_finished({piece_index_t{1}, 1}, NULL); + p->mark_as_finished({piece_index_t{1}, 2}, NULL); + p->mark_as_finished({piece_index_t{1}, 3}, NULL); + + TEST_CHECK(!p->is_seeding()); + p->piece_passed(piece_index_t{1}); + TEST_CHECK(p->is_seeding()); +} + //TODO: 2 test picking with partial pieces and other peers present so that both // backup_pieces and backup_pieces2 are used