From 41808f8742649d021b3a3f76617a048497deb07d Mon Sep 17 00:00:00 2001 From: Arvid Norberg Date: Sat, 6 Sep 2008 21:04:57 +0000 Subject: [PATCH] big change to improve piece picker. Better end-game mode. More options on which pieces to pick. --- include/libtorrent/peer_connection.hpp | 4 +- include/libtorrent/piece_picker.hpp | 114 ++- include/libtorrent/session_settings.hpp | 5 + src/peer_connection.cpp | 50 +- src/piece_picker.cpp | 966 +++++++++++++++--------- src/policy.cpp | 10 +- src/torrent.cpp | 15 +- test/test_piece_picker.cpp | 716 +++++++++++------- 8 files changed, 1205 insertions(+), 675 deletions(-) diff --git a/include/libtorrent/peer_connection.hpp b/include/libtorrent/peer_connection.hpp index cb4f30e40..c9fd1ec5c 100644 --- a/include/libtorrent/peer_connection.hpp +++ b/include/libtorrent/peer_connection.hpp @@ -151,7 +151,7 @@ namespace libtorrent policy::peer* peer_info_struct() const { return m_peer_info; } - enum peer_speed_t { slow, medium, fast }; + enum peer_speed_t { slow = 1, medium, fast }; peer_speed_t peer_speed(); void send_allowed_set(); @@ -187,6 +187,8 @@ namespace libtorrent bool on_parole() const { return peer_info_struct() && peer_info_struct()->on_parole; } + int picker_options() const; + void prefer_whole_pieces(int num) { m_prefer_whole_pieces = num; } diff --git a/include/libtorrent/piece_picker.hpp b/include/libtorrent/piece_picker.hpp index 5bcae3c96..f9f4985e4 100644 --- a/include/libtorrent/piece_picker.hpp +++ b/include/libtorrent/piece_picker.hpp @@ -88,6 +88,14 @@ namespace libtorrent { public: + enum + { + // the number of priority levels + priority_levels = 8, + // priority factor + prio_factor = priority_levels - 4 + }; + struct block_info { block_info(): peer(0), num_peers(0), state(state_none) {} @@ -110,6 +118,20 @@ namespace libtorrent enum piece_state_t { none, slow, medium, fast }; + enum options_t + { + // pick rarest first + rarest_first = 1, + // pick the most common first, or the last pieces if sequential + reverse = 2, + // only pick pieces exclusively requested from this peer + on_parole = 4, + // always pick partial pieces before any other piece + prioritize_partials = 8, + // pick pieces in sequential order + sequential = 16 + }; + struct downloading_piece { downloading_piece(): finished(0), writing(0), requested(0) {} @@ -133,9 +155,6 @@ namespace libtorrent void get_availability(std::vector& avail) const; - void sequential_download(bool sd); - bool sequential_download() const { return m_sequential_download >= 0; } - // increases the peer count for the given piece // (is used when a HAVE message is received) void inc_refcount(int index); @@ -161,6 +180,9 @@ namespace libtorrent void we_have(int index); void we_dont_have(int index); + int cursor() const { return m_cursor; } + int reverse_cursor() const { return m_reverse_cursor; } + // sets all pieces to dont-have void init(int blocks_per_piece, int total_num_blocks); int num_pieces() const { return int(m_piece_map.size()); } @@ -202,11 +224,9 @@ namespace libtorrent // The last argument is the policy::peer pointer for the peer that // we'll download from. void pick_pieces(bitfield const& pieces - , std::vector& interesting_blocks - , int num_pieces, int prefer_whole_pieces - , void* peer, piece_state_t speed - , bool rarest_first, bool on_parole - , std::vector const& suggested_pieces) const; + , std::vector& interesting_blocks, int num_blocks + , int prefer_whole_pieces, void* peer, piece_state_t speed + , int options, std::vector const& suggested_pieces) const; // picks blocks from each of the pieces in the piece_list // vector that is also in the piece bitmask. The blocks @@ -214,20 +234,23 @@ namespace libtorrent // added to backup_blocks. num blocks is the number of // blocks to be picked. Blocks are not picked from pieces // that are being downloaded - int add_blocks(std::vector const& piece_list - , bitfield const& pieces - , std::vector& interesting_blocks - , int num_blocks, int prefer_whole_pieces - , void* peer, std::vector const& ignore) const; - - // picks blocks only from downloading pieces - int add_blocks_downloading( - bitfield const& pieces + int add_blocks(int piece, bitfield const& pieces , std::vector& interesting_blocks , std::vector& backup_blocks + , std::vector& backup_blocks2 + , int num_blocks, int prefer_whole_pieces + , void* peer, std::vector const& ignore + , piece_state_t speed, int options) const; + + // picks blocks only from downloading pieces + int add_blocks_downloading(downloading_piece const& dp + , bitfield const& pieces + , std::vector& interesting_blocks + , std::vector& backup_blocks + , std::vector& backup_blocks2 , int num_blocks, int prefer_whole_pieces , void* peer, piece_state_t speed - , bool on_parole) const; + , int options) const; // clears the peer pointer in all downloading pieces with this // peer pointer @@ -291,6 +314,8 @@ namespace libtorrent void check_invariant(const torrent* t = 0) const; void verify_pick(std::vector const& picked , bitfield const& bits) const; +#endif +#if defined TORRENT_PICKER_LOG || !defined NDEBUG void print_pieces() const; #endif @@ -313,6 +338,7 @@ namespace libtorrent friend struct piece_pos; bool can_pick(int piece, bitfield const& bitmask) const; + bool is_piece_free(int piece, bitfield const& bitmask) const; std::pair expand_piece(int piece, int whole_pieces , bitfield const& have) const; @@ -354,7 +380,7 @@ namespace libtorrent // the priority value that means the piece is filtered filter_priority = 0, // the max number the peer count can hold - max_peer_count = 0x3ff + max_peer_count = 0x3ff, }; bool have() const { return index == we_have_index; } @@ -364,20 +390,41 @@ namespace libtorrent bool filtered() const { return piece_priority == filter_priority; } void filtered(bool f) { piece_priority = f ? filter_priority : 0; } + // prio 7 is always top priority + // prio 0 is always -1 (don't pick) + // downloading pieces are always on an even prio_factor priority + // + // availability x, downloading + // | availability x, prio 3; availability 2x, prio 6 + // | | availability x, prio 2; availability 2x, prio 5 + // | | | availability x, prio 1; availability 2x, prio 4 + // | | | | + // +---+---+---+---+ + // | 0 | 1 | 2 | 3 | + // +---+---+---+---+ + int priority(piece_picker const* picker) const { - if (downloading || filtered() - || have() || peer_count + picker->m_seeds == 0) + // filtered pieces (prio = 0), pieces we have or pieces with + // availability = 0 should not be present in the piece list + // returning -1 indicates that they shouldn't. + if (filtered() || have() || peer_count + picker->m_seeds == 0) return -1; - // priority 5, 6 and 7 disregards availability of the piece - if (piece_priority > 4) return 7 - piece_priority; + // prio 7 disregards availability + if (piece_priority == priority_levels - 1) return 1 - downloading; - // pieces we are currently downloading have high priority - int prio = peer_count * 4; -// if (prio >= picker->m_prio_limit * 6) prio = picker->m_prio_limit * 6; + // prio 4,5,6 halves the availability of a piece + int availability = peer_count; + int priority = piece_priority; + if (piece_priority >= priority_levels / 2) + { + availability /= 2; + priority -= (priority_levels - 2) / 2; + } - return prio + (4 - piece_priority); + if (downloading) return availability * prio_factor; + return availability * prio_factor + (priority_levels / 2) - priority; } bool operator!=(piece_pos p) const @@ -467,11 +514,14 @@ namespace libtorrent // the number of pieces we have int m_num_have; - // -1 means sequential download is not active. - // >= 0 means that pieces are requested in sequential order - // and this variable is the next piece to request. - // in that case m_pieces is cleared and not used. - int m_sequential_download; + // we have all pieces in the range [0, m_cursor) + // m_cursor is the first piece we don't have + int m_cursor; + + // we have all pieces in the range [m_reverse_cursor, end) + // m_reverse_cursor is the first piece where we also have + // all the subsequent pieces + int m_reverse_cursor; // if this is set to true, it means update_pieces() // has to be called before accessing m_pieces. diff --git a/include/libtorrent/session_settings.hpp b/include/libtorrent/session_settings.hpp index 9aafbb921..071e0d5c0 100644 --- a/include/libtorrent/session_settings.hpp +++ b/include/libtorrent/session_settings.hpp @@ -141,6 +141,7 @@ namespace libtorrent , auto_scrape_min_interval(300) , max_peerlist_size(8000) , min_announce_interval(5 * 60) + , prioritize_partial_pieces(false) {} // this is the user agent that will be sent to the tracker @@ -436,6 +437,10 @@ namespace libtorrent // that is lower than this, will be clamped to this // value. It's specified in seconds int min_announce_interval; + + // if true, partial pieces are picked before pieces + // that are more rare + bool prioritize_partial_pieces; }; #ifndef TORRENT_DISABLE_DHT diff --git a/src/peer_connection.cpp b/src/peer_connection.cpp index 9fa4c88b1..138a21b46 100644 --- a/src/peer_connection.cpp +++ b/src/peer_connection.cpp @@ -572,6 +572,50 @@ namespace libtorrent #endif } + int peer_connection::picker_options() const + { + int ret = 0; + boost::shared_ptr t = m_torrent.lock(); + TORRENT_ASSERT(t); + if (!t) return 0; + + if (t->is_sequential_download()) + { + ret |= piece_picker::sequential; + } + else if (t->num_have() < t->settings().initial_picker_threshold) + { + // if we have fewer pieces than a certain threshols + // don't pick rare pieces, just pick random ones, + // and prioritize finishing them + ret |= piece_picker::prioritize_partials; + } + else + { + ret |= piece_picker::rarest_first; + } + + if (m_snubbed) + { + // snubbed peers should request + // the common pieces first, just to make + // it more likely for all snubbed peers to + // request blocks from the same piece + ret |= piece_picker::reverse; + } + + if (t->settings().prioritize_partial_pieces) + ret |= piece_picker::prioritize_partials; + + if (on_parole()) ret |= piece_picker::on_parole + | piece_picker::prioritize_partials; + + // only one of rarest_first, common_first and sequential can be set. + TORRENT_ASSERT(bool(ret & piece_picker::rarest_first) + + bool(ret & piece_picker::sequential) <= 1); + return ret; + } + void peer_connection::fast_reconnect(bool r) { if (!peer_info_struct() || peer_info_struct()->fast_reconnects > 1) @@ -1724,9 +1768,12 @@ namespace libtorrent bool multi = picker.num_peers(block_finished) > 1; picker.mark_as_writing(block_finished, peer_info_struct()); + TORRENT_ASSERT(picker.num_peers(block_finished) == 0); // if we requested this block from other peers, cancel it now if (multi) t->cancel_block(block_finished); + TORRENT_ASSERT(picker.num_peers(block_finished) == 0); + #if !defined NDEBUG && !defined TORRENT_DISABLE_INVARIANT_CHECKS t->check_invariant(); #endif @@ -1777,6 +1824,7 @@ namespace libtorrent TORRENT_ASSERT(p.piece == j.piece); TORRENT_ASSERT(p.start == j.offset); + TORRENT_ASSERT(picker.num_peers(block_finished) == 0); picker.mark_as_finished(block_finished, peer_info_struct()); if (t->alerts().should_post()) { @@ -3899,7 +3947,7 @@ namespace libtorrent m_speed = medium; else if (download_rate < torrent_download_rate / 15 && m_speed == fast) m_speed = medium; - else if (download_rate < torrent_download_rate / 63 && m_speed == medium) + else m_speed = slow; return m_speed; diff --git a/src/piece_picker.cpp b/src/piece_picker.cpp index bb0b460a3..f2ee3d450 100644 --- a/src/piece_picker.cpp +++ b/src/piece_picker.cpp @@ -46,9 +46,9 @@ POSSIBILITY OF SUCH DAMAGE. #include "libtorrent/torrent.hpp" #endif -//#define TORRENT_PIECE_PICKER_INVARIANT_CHECK INVARIANT_CHECK +#define TORRENT_PIECE_PICKER_INVARIANT_CHECK INVARIANT_CHECK //#define TORRENT_NO_EXPENSIVE_INVARIANT_CHECK -#define TORRENT_PIECE_PICKER_INVARIANT_CHECK +//#define TORRENT_PIECE_PICKER_INVARIANT_CHECK //#define TORRENT_PICKER_LOG @@ -58,10 +58,13 @@ namespace libtorrent piece_picker::piece_picker() : m_seeds(0) , m_priority_boundries(1, int(m_pieces.size())) + , m_blocks_per_piece(0) + , m_blocks_in_last_piece(0) , m_num_filtered(0) , m_num_have_filtered(0) , m_num_have(0) - , m_sequential_download(-1) + , m_cursor(0) + , m_reverse_cursor(0) , m_dirty(false) { #ifdef TORRENT_PICKER_LOG @@ -77,10 +80,15 @@ namespace libtorrent TORRENT_ASSERT(blocks_per_piece > 0); TORRENT_ASSERT(total_num_blocks >= 0); +#ifdef TORRENT_PICKER_LOG + std::cerr << "piece_picker::init()" << std::endl; +#endif // allocate the piece_map to cover all pieces // and make them invalid (as if we don't have a single piece) m_piece_map.resize((total_num_blocks + blocks_per_piece-1) / blocks_per_piece , piece_pos(0, 0)); + m_reverse_cursor = int(m_piece_map.size()); + m_cursor = 0; m_num_filtered += m_num_have_filtered; m_num_have_filtered = 0; @@ -94,10 +102,16 @@ namespace libtorrent i->index = 0; } + for (std::vector::const_iterator i = m_piece_map.begin() + m_cursor + , end(m_piece_map.end()); i != end && (i->have() || i->filtered()); + ++i, ++m_cursor); + for (std::vector::const_iterator i = m_piece_map.begin() + + m_reverse_cursor - 1; m_reverse_cursor > 0 && (i->have() || i->filtered()); + --i, --m_reverse_cursor); + // the piece index is stored in 20 bits, which limits the allowed // number of pieces somewhat - if (m_piece_map.size() >= piece_pos::we_have_index) - throw std::runtime_error("too many pieces in torrent"); + TORRENT_ASSERT(m_piece_map.size() < piece_pos::we_have_index); m_blocks_per_piece = blocks_per_piece; m_blocks_in_last_piece = total_num_blocks % blocks_per_piece; @@ -106,30 +120,6 @@ namespace libtorrent TORRENT_ASSERT(m_blocks_in_last_piece <= m_blocks_per_piece); } - void piece_picker::sequential_download(bool sd) - { - if (sd == sequential_download()) return; - - TORRENT_PIECE_PICKER_INVARIANT_CHECK; - - if (sd) - { - std::vector().swap(m_pieces); - std::vector().swap(m_priority_boundries); - - // initialize m_sdquential_download - m_sequential_download = 0; - for (std::vector::const_iterator i = m_piece_map.begin() - , end(m_piece_map.end()); i != end && (i->have() || i->filtered()); - ++i, ++m_sequential_download); - } - else - { - m_sequential_download = -1; - m_dirty = true; - } - } - void piece_picker::piece_info(int index, piece_picker::downloading_piece& st) const { TORRENT_PIECE_PICKER_INVARIANT_CHECK; @@ -234,7 +224,7 @@ namespace libtorrent } } -#ifdef TORRENT_PICKER_LOG +#if defined TORRENT_PICKER_LOG || !defined NDEBUG void piece_picker::print_pieces() const { for (std::vector::const_iterator i = m_priority_boundries.begin() @@ -314,11 +304,19 @@ namespace libtorrent TORRENT_ASSERT(num_writing == i->writing); TORRENT_ASSERT(num_finished == i->finished); } + int num_pieces = int(m_piece_map.size()); + TORRENT_ASSERT(m_cursor >= 0); + TORRENT_ASSERT(m_cursor <= num_pieces); + TORRENT_ASSERT(m_reverse_cursor <= num_pieces); + TORRENT_ASSERT(m_reverse_cursor >= 0); + TORRENT_ASSERT(m_reverse_cursor > m_cursor + || (m_cursor == num_pieces && m_reverse_cursor == 0)); + #ifdef TORRENT_NO_EXPENSIVE_INVARIANT_CHECK return; #endif - if (m_sequential_download == -1 && !m_dirty) + if (!m_dirty) { TORRENT_ASSERT(!m_priority_boundries.empty()); int prio = 0; @@ -332,13 +330,24 @@ namespace libtorrent } TORRENT_ASSERT(m_priority_boundries.back() == int(m_pieces.size())); } - else if (m_sequential_download >= 0) + int index = 0; + for (std::vector::const_iterator i = m_piece_map.begin() + , end(m_piece_map.end()); i != end && (i->have() || i->filtered()); + ++i, ++index); + TORRENT_ASSERT(m_cursor == index); + index = num_pieces; + if (num_pieces > 0) { - int index = 0; for (std::vector::const_iterator i = m_piece_map.begin() - , end(m_piece_map.end()); i != end && (i->have() || i->filtered()); - ++i, ++index); - TORRENT_ASSERT(m_sequential_download == index); + + (index - 1); index > 0 && (i->have() || i->filtered()); --i, --index); + TORRENT_ASSERT(index == num_pieces + || m_piece_map[index].have() + || m_piece_map[index].filtered()); + TORRENT_ASSERT(m_reverse_cursor == index); + } + else + { + TORRENT_ASSERT(m_reverse_cursor == 0); } int num_filtered = 0; @@ -404,9 +413,11 @@ namespace libtorrent if (t != 0) TORRENT_ASSERT(!t->have_piece(index)); - if (m_sequential_download == -1 && !m_dirty) + int prio = p.priority(this); + TORRENT_ASSERT(prio == -1 || p.downloading == (prio % piece_picker::prio_factor == 0)); + + if (!m_dirty) { - int prio = p.priority(this); TORRENT_ASSERT(prio < int(m_priority_boundries.size()) || m_dirty); if (prio >= 0) @@ -424,11 +435,6 @@ namespace libtorrent == m_pieces.end()); } } - else if (m_sequential_download >= 0) - { - TORRENT_ASSERT(m_pieces.empty()); - TORRENT_ASSERT(m_priority_boundries.empty()); - } int count = std::count_if(m_downloads.begin(), m_downloads.end() , has_index(index)); @@ -512,7 +518,6 @@ namespace libtorrent piece_pos& p = m_piece_map[index]; TORRENT_ASSERT(!p.filtered()); TORRENT_ASSERT(!p.have()); - TORRENT_ASSERT(m_sequential_download == -1); int priority = p.priority(this); TORRENT_ASSERT(priority >= 0); @@ -564,6 +569,10 @@ namespace libtorrent #ifdef TORRENT_PICKER_LOG print_pieces(); +#endif +// shuffle(priority, new_index); +#ifdef TORRENT_PICKER_LOG +// print_pieces(); #endif } } @@ -572,7 +581,6 @@ namespace libtorrent { TORRENT_ASSERT(!m_dirty); TORRENT_ASSERT(priority >= 0); - TORRENT_ASSERT(m_sequential_download == -1); #ifdef TORRENT_PICKER_LOG std::cerr << "remove " << m_pieces[elem_index] << " (" << priority << ")" << std::endl; @@ -618,7 +626,6 @@ namespace libtorrent TORRENT_ASSERT(!m_dirty); TORRENT_ASSERT(priority >= 0); TORRENT_ASSERT(elem_index >= 0); - TORRENT_ASSERT(m_sequential_download == -1); TORRENT_ASSERT(int(m_priority_boundries.size()) > priority); @@ -721,10 +728,15 @@ namespace libtorrent void piece_picker::shuffle(int priority, int elem_index) { +#ifdef TORRENT_PICKER_LOG + std::cerr << "shuffle()" << std::endl; +#endif + TORRENT_ASSERT(!m_dirty); TORRENT_ASSERT(priority >= 0); TORRENT_ASSERT(elem_index >= 0); - TORRENT_ASSERT(m_sequential_download == -1); + TORRENT_ASSERT(elem_index < int(m_pieces.size())); + TORRENT_ASSERT(m_piece_map[m_pieces[elem_index]].priority(this) == priority); int range_start, range_end; priority_range(priority, &range_start, &range_end); @@ -773,6 +785,14 @@ namespace libtorrent , has_index(index)); TORRENT_ASSERT(i != m_downloads.end()); +#ifndef NDEBUG + int num_blocks = blocks_in_piece(i->index); + for (int k = 0; k < num_blocks; ++k) + { + TORRENT_ASSERT(i->info[k].state == block_info::state_finished); + TORRENT_ASSERT(i->info[k].num_peers == 0); + } +#endif erase_download_piece(i); piece_pos& p = m_piece_map[index]; @@ -782,11 +802,6 @@ namespace libtorrent if (new_priority == prev_priority) return; if (m_dirty) return; - if (m_sequential_download >= 0) - { - m_dirty = true; - return; - } if (prev_priority == -1) { add(index); @@ -840,15 +855,11 @@ namespace libtorrent void piece_picker::inc_refcount(int index) { +#ifdef TORRENT_EXPENSIVE_INVARIANT_CHECKS TORRENT_PIECE_PICKER_INVARIANT_CHECK; +#endif piece_pos& p = m_piece_map[index]; - if (m_sequential_download >= 0) - { - ++p.peer_count; - m_dirty = true; - return; - } int prev_priority = p.priority(this); ++p.peer_count; @@ -863,16 +874,11 @@ namespace libtorrent void piece_picker::dec_refcount(int index) { +#ifdef TORRENT_EXPENSIVE_INVARIANT_CHECKS TORRENT_PIECE_PICKER_INVARIANT_CHECK; +#endif piece_pos& p = m_piece_map[index]; - if (m_sequential_download >= 0) - { - TORRENT_ASSERT(p.peer_count > 0); - --p.peer_count; - m_dirty = true; - return; - } int prev_priority = p.priority(this); TORRENT_ASSERT(p.peer_count > 0); --p.peer_count; @@ -897,7 +903,7 @@ namespace libtorrent } } - if (updated && m_sequential_download == -1) m_dirty = true; + if (updated) m_dirty = true; } void piece_picker::dec_refcount(bitfield const& bitmask) @@ -917,13 +923,12 @@ namespace libtorrent } } - if (updated && m_sequential_download == -1) m_dirty = true; + if (updated) m_dirty = true; } void piece_picker::update_pieces() const { TORRENT_ASSERT(m_dirty); - TORRENT_ASSERT(m_sequential_download == -1); if (m_priority_boundries.empty()) m_priority_boundries.resize(1, 0); #ifdef TORRENT_PICKER_LOG std::cerr << "update_pieces" << std::endl; @@ -1004,16 +1009,29 @@ namespace libtorrent piece_pos& p = m_piece_map[index]; TORRENT_ASSERT(p.downloading == 0); +#ifdef TORRENT_PICKER_LOG + std::cerr << "piece_picker::we_dont_have(" << index << ")" << std::endl; +#endif if (!p.have()) return; - if (m_sequential_download > index) - m_sequential_download = index; - if (p.filtered()) { ++m_num_filtered; --m_num_have_filtered; } + else + { + // update cursors + if (index < m_cursor) + m_cursor = index; + if (index >= m_reverse_cursor) + m_reverse_cursor = index + 1; + if (m_reverse_cursor == m_cursor) + { + m_reverse_cursor = 0; + m_cursor = num_pieces(); + } + } --m_num_have; p.set_not_have(); @@ -1032,10 +1050,13 @@ namespace libtorrent TORRENT_ASSERT(index >= 0); TORRENT_ASSERT(index < (int)m_piece_map.size()); +#ifdef TORRENT_PICKER_LOG + std::cerr << "piece_picker::we_have(" << index << ")" << std::endl; +#endif piece_pos& p = m_piece_map[index]; int info_index = p.index; int priority = p.priority(this); - TORRENT_ASSERT(priority < int(m_priority_boundries.size())); + TORRENT_ASSERT(priority < int(m_priority_boundries.size()) || m_dirty); if (p.downloading) { @@ -1051,13 +1072,6 @@ namespace libtorrent TORRENT_ASSERT(std::find_if(m_downloads.begin(), m_downloads.end() , has_index(index)) == m_downloads.end()); - if (m_sequential_download == index) - { - ++m_sequential_download; - for (std::vector::const_iterator i = m_piece_map.begin() + m_sequential_download - , end(m_piece_map.end()); i != end && (i->have() || i->filtered()); - ++i, ++m_sequential_download); - } if (p.have()) return; if (p.filtered()) { @@ -1066,6 +1080,33 @@ namespace libtorrent } ++m_num_have; p.set_have(); + if (m_cursor == m_reverse_cursor - 1 && + m_cursor == index) + { + m_cursor = int(m_piece_map.size()); + m_reverse_cursor = 0; + TORRENT_ASSERT(num_pieces() > 0); + } + else if (m_cursor == index) + { + ++m_cursor; + for (std::vector::const_iterator i = m_piece_map.begin() + m_cursor + , end(m_piece_map.end()); i != end && (i->have() || i->filtered()); + ++i, ++m_cursor); + } + else if (m_reverse_cursor - 1 == index) + { + --m_reverse_cursor; + TORRENT_ASSERT(m_piece_map[m_reverse_cursor].have() + || m_piece_map[m_reverse_cursor].filtered()); + for (std::vector::const_iterator i = m_piece_map.begin() + + m_reverse_cursor - 1; m_reverse_cursor > 0 && (i->have() || i->filtered()); + --i, --m_reverse_cursor); + TORRENT_ASSERT(m_piece_map[m_reverse_cursor].have() + || m_piece_map[m_reverse_cursor].filtered()); + } + TORRENT_ASSERT(m_reverse_cursor > m_cursor + || (m_cursor == num_pieces() && m_reverse_cursor == 0)); if (priority == -1) return; if (m_dirty) return; remove(priority, info_index); @@ -1074,7 +1115,9 @@ namespace libtorrent bool piece_picker::set_piece_priority(int index, int new_piece_priority) { +#ifdef TORRENT_EXPENSIVE_INVARIANT_CHECKS TORRENT_PIECE_PICKER_INVARIANT_CHECK; +#endif TORRENT_ASSERT(new_piece_priority >= 0); TORRENT_ASSERT(new_piece_priority <= 7); TORRENT_ASSERT(index >= 0); @@ -1093,16 +1136,61 @@ namespace libtorrent && p.piece_priority != piece_pos::filter_priority) { // the piece just got filtered - if (p.have()) ++m_num_have_filtered; - else ++m_num_filtered; + if (p.have()) + { + ++m_num_have_filtered; + } + else + { + ++m_num_filtered; + + // update m_cursor + if (m_cursor == m_reverse_cursor - 1 && m_cursor == index) + { + m_cursor = int(m_piece_map.size()); + m_reverse_cursor = 0; + } + else if (m_cursor == index) + { + ++m_cursor; + while (m_cursor < int(m_piece_map.size()) + && (m_piece_map[m_cursor].have() + || m_piece_map[m_cursor].filtered())) + ++m_cursor; + } + else if (m_reverse_cursor == index + 1) + { + --m_reverse_cursor; + while (m_reverse_cursor > 0 + && (m_piece_map[m_reverse_cursor-1].have() + || m_piece_map[m_reverse_cursor-1].filtered())) + --m_reverse_cursor; + } + } ret = true; } else if (new_piece_priority != piece_pos::filter_priority && p.piece_priority == piece_pos::filter_priority) { // the piece just got unfiltered - if (p.have()) --m_num_have_filtered; - else --m_num_filtered; + if (p.have()) + { + --m_num_have_filtered; + } + else + { + --m_num_filtered; + // update cursors + if (index < m_cursor) + m_cursor = index; + if (index >= m_reverse_cursor) + m_reverse_cursor = index + 1; + if (m_reverse_cursor == m_cursor) + { + m_reverse_cursor = 0; + m_cursor = num_pieces(); + } + } ret = true; } TORRENT_ASSERT(m_num_filtered >= 0); @@ -1159,6 +1247,25 @@ namespace libtorrent // ============ end deprecation ============== + namespace + { + int append_blocks(std::vector& dst, std::vector& src + , int num_blocks) + { + if (src.empty()) return num_blocks; + int to_copy; +// if (prefer_whole_pieces == 0) + to_copy = (std::min)(int(src.size()), num_blocks); +// else +// to_copy = int(src.size()); + + dst.insert(dst.end() + , src.begin(), src.begin() + to_copy); + src.clear(); + return num_blocks - to_copy; + } + } + // pieces describes which pieces the peer we're requesting from // has. // interesting_blocks is an out parameter, and will be filled @@ -1174,18 +1281,37 @@ namespace libtorrent // to pick blocks from the same pieces as fast peers, and vice // versa. Downloading pieces are marked as being fast, medium // or slow once they're started. + + // options are: + // * rarest_first + // pick the rarest pieces first + // * reverse + // reverse the piece picking. Pick the most common + // pieces first or the last pieces (if picking sequential) + // * sequential + // download pieces in-order + // * on_parole + // the peer is on parole, only pick whole pieces which + // has only been downloaded and requested from the same + // peer + // * prioritize_partials + // pick blocks from downloading pieces first + + // only one of rarest_first, sequential can be set + void piece_picker::pick_pieces(bitfield const& pieces - , std::vector& interesting_blocks - , int num_blocks, int prefer_whole_pieces - , void* peer, piece_state_t speed, bool rarest_first - , bool on_parole, std::vector const& suggested_pieces) const + , std::vector& interesting_blocks, int num_blocks + , int prefer_whole_pieces, void* peer, piece_state_t speed + , int options, std::vector const& suggested_pieces) const { + // only one of rarest_first and sequential can be set. + TORRENT_ASSERT(bool(options & rarest_first) + + bool(options & sequential) <= 1); TORRENT_PIECE_PICKER_INVARIANT_CHECK; TORRENT_ASSERT(num_blocks > 0); TORRENT_ASSERT(pieces.size() == m_piece_map.size()); TORRENT_ASSERT(!m_priority_boundries.empty() - || m_sequential_download >= 0 || m_dirty); // this will be filled with blocks that we should not request @@ -1195,6 +1321,7 @@ namespace libtorrent // blocks belonging to a piece that others have // downloaded to std::vector backup_blocks; + std::vector backup_blocks2; const std::vector empty_vector; // When prefer_whole_pieces is set (usually set when downloading from @@ -1202,43 +1329,122 @@ namespace libtorrent // ignored as long as possible. All blocks found in downloading // pieces are regarded as backup blocks - num_blocks = add_blocks_downloading(pieces - , interesting_blocks, backup_blocks, num_blocks - , prefer_whole_pieces, peer, speed, on_parole); + if (options & prioritize_partials) + { + for (std::vector::const_iterator i = m_downloads.begin() + , end(m_downloads.end()); i != end; ++i) + { + if (!pieces[i->index]) continue; + num_blocks = add_blocks_downloading(*i, pieces + , interesting_blocks, backup_blocks, backup_blocks2 + , num_blocks, prefer_whole_pieces, peer, speed, options); + if (num_blocks <= 0) return; + } - if (num_blocks <= 0) return; + num_blocks = append_blocks(interesting_blocks, backup_blocks + , num_blocks); + if (num_blocks <= 0) return; + + num_blocks = append_blocks(interesting_blocks, backup_blocks2 + , num_blocks); + if (num_blocks <= 0) return; + } if (!suggested_pieces.empty()) { - num_blocks = add_blocks(suggested_pieces, pieces - , interesting_blocks, num_blocks - , prefer_whole_pieces, peer, empty_vector); - if (num_blocks == 0) return; + for (std::vector::const_iterator i = suggested_pieces.begin(); + i != suggested_pieces.end(); ++i) + { + if (!is_piece_free(*i, pieces)) continue; + num_blocks = add_blocks(*i, pieces + , interesting_blocks, backup_blocks + , backup_blocks2, num_blocks + , prefer_whole_pieces, peer, empty_vector + , speed, options); + if (num_blocks <= 0) return; + } } - if (m_sequential_download >= 0) + if (options & sequential) { - for (int i = m_sequential_download; - i < int(m_piece_map.size()) && num_blocks > 0; ++i) + if (options & reverse) { - if (!can_pick(i, pieces)) continue; - int num_blocks_in_piece = blocks_in_piece(i); - if (prefer_whole_pieces == 0 && num_blocks_in_piece > num_blocks) - num_blocks_in_piece = num_blocks; - for (int j = 0; j < num_blocks_in_piece; ++j) - { - interesting_blocks.push_back(piece_block(i, j)); - --num_blocks; + for (int i = m_reverse_cursor - 1; i >= m_cursor; --i) + { + if (!is_piece_free(i, pieces)) continue; + num_blocks = add_blocks(i, pieces + , interesting_blocks, backup_blocks + , backup_blocks2, num_blocks + , prefer_whole_pieces, peer, suggested_pieces + , speed, options); + if (num_blocks <= 0) return; + } + } + else + { + for (int i = m_cursor; i < m_reverse_cursor; ++i) + { + if (!is_piece_free(i, pieces)) continue; + num_blocks = add_blocks(i, pieces + , interesting_blocks, backup_blocks + , backup_blocks2, num_blocks + , prefer_whole_pieces, peer, suggested_pieces + , speed, options); + if (num_blocks <= 0) return; } } } - else if (rarest_first) + else if (options & rarest_first) { if (m_dirty) update_pieces(); - num_blocks = add_blocks(m_pieces, pieces - , interesting_blocks, num_blocks - , prefer_whole_pieces, peer, suggested_pieces); - TORRENT_ASSERT(num_blocks >= 0); + TORRENT_ASSERT(!m_dirty); + + if (options & reverse) + { + // it's a bit complicated in order to always prioritize + // partial pieces, and respect priorities. Every chunk + // of 4 priority levels are traversed in forward order, but otherwise + // they are traversed in reverse order + // round up to an even 4 priority boundry, to make it simpler + // to do the akward reverse traversing +#define div_round_up(n, d) (((n) + (d) - 1) / (d)) + m_priority_boundries.resize(div_round_up(m_priority_boundries.size() + , prio_factor) * prio_factor, m_priority_boundries.back()); + for (int i = m_priority_boundries.size() - 1; i >= 0; --i) + { + int prio = (i / prio_factor) * prio_factor + + prio_factor - 1 - (i % prio_factor); + + TORRENT_ASSERT(prio >= 0); + TORRENT_ASSERT(prio < int(m_priority_boundries.size())); + int start = prio == 0 ? 0 : m_priority_boundries[prio - 1]; + for (int p = start; p < m_priority_boundries[prio]; ++p) + { + if (!is_piece_free(m_pieces[p], pieces)) continue; + num_blocks = add_blocks(m_pieces[p], pieces + , interesting_blocks, backup_blocks + , backup_blocks2, num_blocks + , prefer_whole_pieces, peer, suggested_pieces + , speed, options); + if (num_blocks <= 0) return; + } + } +#undef div_round_up + } + else + { + for (std::vector::const_iterator i = m_pieces.begin(); + i != m_pieces.end(); ++i) + { + if (!is_piece_free(*i, pieces)) continue; + num_blocks = add_blocks(*i, pieces + , interesting_blocks, backup_blocks + , backup_blocks2, num_blocks + , prefer_whole_pieces, peer, suggested_pieces + , speed, options); + if (num_blocks <= 0) return; + } + } } else { @@ -1247,24 +1453,23 @@ namespace libtorrent // pieces are) int start_piece = rand() % m_piece_map.size(); - // if we have suggested pieces, try to find one of those instead - for (std::vector::const_iterator i = suggested_pieces.begin() - , end(suggested_pieces.end()); i != end; ++i) - { - if (!can_pick(*i, pieces)) continue; - start_piece = *i; - break; - } int piece = start_piece; while (num_blocks > 0) { - while (!can_pick(piece, pieces)) + bool done = false; + // skip pieces we can't pick, and suggested pieces + // since we've already picked those + while (!can_pick(piece, pieces) + && std::find(suggested_pieces.begin() + , suggested_pieces.end(), piece) + == suggested_pieces.end()) { ++piece; if (piece == int(m_piece_map.size())) piece = 0; // could not find any more pieces - if (piece == start_piece) return; + if (piece == start_piece) { done = true; break; } } + if (done) break; int start, end; boost::tie(start, end) = expand_piece(piece, prefer_whole_pieces, pieces); @@ -1284,16 +1489,125 @@ namespace libtorrent piece = end; if (piece == int(m_piece_map.size())) piece = 0; // could not find any more pieces - if (piece == start_piece) return; + if (piece == start_piece) break; } } if (num_blocks <= 0) return; - if (!backup_blocks.empty()) - interesting_blocks.insert(interesting_blocks.end() - , backup_blocks.begin(), backup_blocks.end()); +#ifndef NDEBUG + verify_pick(interesting_blocks, pieces); + verify_pick(backup_blocks, pieces); + verify_pick(backup_blocks2, pieces); +#endif + + num_blocks = append_blocks(interesting_blocks, backup_blocks + , num_blocks); + if (num_blocks <= 0) return; + + num_blocks = append_blocks(interesting_blocks, backup_blocks2 + , num_blocks); + if (num_blocks <= 0) return; + + // don't double-pick anything if the peer is on parole + if (options & on_parole) return; + + for (std::vector::const_iterator i = m_downloads.begin() + , end(m_downloads.end()); i != end; ++i) + { + if (!pieces[i->index]) continue; + + int num_blocks_in_piece = blocks_in_piece(i->index); + + // fill in with blocks requested from other peers + // as backups + for (int j = 0; j < num_blocks_in_piece; ++j) + { + block_info const& info = i->info[j]; + if (info.state != block_info::state_requested + || info.peer == peer) + continue; + interesting_blocks.push_back(piece_block(i->index, j)); + } + } + +#ifndef NDEBUG +// make sure that we at this point have added requests to all unrequested blocks +// in all downloading pieces + + for (std::vector::const_iterator i = m_downloads.begin() + , end(m_downloads.end()); i != end; ++i) + { + if (!pieces[i->index]) continue; + + int num_blocks_in_piece = blocks_in_piece(i->index); + for (int j = 0; j < num_blocks_in_piece; ++j) + { + block_info const& info = i->info[j]; + if (info.state != block_info::state_none) continue; + std::vector::iterator k = std::find( + interesting_blocks.begin(), interesting_blocks.end() + , piece_block(i->index, j)); + if (k != interesting_blocks.end()) continue; + + std::cerr << "interesting blocks:" << std::endl; + for (k = interesting_blocks.begin(); k != interesting_blocks.end(); ++k) + std::cerr << "(" << k->piece_index << ", " << k->block_index << ") "; + std::cerr << std::endl; + std::cerr << "num_blocks: " << num_blocks << std::endl; + + for (std::vector::const_iterator l = m_downloads.begin() + , end(m_downloads.end()); l != end; ++l) + { + std::cerr << l->index << " : "; + int num_blocks_in_piece = blocks_in_piece(l->index); + for (int m = 0; m < num_blocks_in_piece; ++m) + std::cerr << l->info[m].state; + std::cerr << std::endl; + } + + TORRENT_ASSERT(false); + } + } + + if (interesting_blocks.empty()) + { +// print_pieces(); + for (int i = 0; i < num_pieces(); ++i) + { + if (!pieces[i]) continue; + if (piece_priority(i) == 0) continue; + if (have_piece(i)) continue; + + std::vector::const_iterator k + = std::find_if(m_downloads.begin(), m_downloads.end(), has_index(i)); + + TORRENT_ASSERT(k != m_downloads.end()); + if (k == m_downloads.end()) continue; + + // this assert is not valid for web_seeds + /* + int num_blocks_in_piece = blocks_in_piece(k->index); + for (int j = 0; j < num_blocks_in_piece; ++j) + { + block_info const& info = k->info[j]; + if (info.state == block_info::state_finished) continue; + TORRENT_ASSERT(info.peer != 0); + } + */ + } + } +#endif + + } + + bool piece_picker::is_piece_free(int piece, bitfield const& bitmask) const + { + TORRENT_ASSERT(piece >= 0 && piece < int(m_piece_map.size())); + return bitmask[piece] + && !m_piece_map[piece].have() + && !m_piece_map[piece].filtered(); } bool piece_picker::can_pick(int piece, bitfield const& bitmask) const @@ -1342,254 +1656,168 @@ namespace libtorrent } } - int piece_picker::add_blocks(std::vector const& piece_list + int piece_picker::add_blocks(int piece , bitfield const& pieces , std::vector& interesting_blocks + , std::vector& backup_blocks + , std::vector& backup_blocks2 , int num_blocks, int prefer_whole_pieces - , void* peer, std::vector const& ignore) const + , void* peer, std::vector const& ignore + , piece_state_t speed, int options) const { - for (std::vector::const_iterator i = piece_list.begin(); - i != piece_list.end(); ++i) + TORRENT_ASSERT(piece >= 0); + TORRENT_ASSERT(piece < (int)m_piece_map.size()); + TORRENT_ASSERT(is_piece_free(piece, pieces)); + +// std::cout << "add_blocks(" << piece << ")" << std::endl; +// std::cout << " num_blocks " << num_blocks << std::endl; + + // ignore pieces found in the ignore list + if (std::find(ignore.begin(), ignore.end(), piece) != ignore.end()) return num_blocks; + + TORRENT_ASSERT(m_piece_map[piece].priority(this) >= 0); + if (m_piece_map[piece].downloading) { - TORRENT_ASSERT(*i >= 0); - TORRENT_ASSERT(*i < (int)m_piece_map.size()); + // if we're prioritizing partials, we've already + // looked through the downloading pieces + if (options & prioritize_partials) return num_blocks; - // if the peer doesn't have the piece - // or if it's set to 0 priority - // skip it - if (!can_pick(*i, pieces)) continue; + std::vector::const_iterator i + = std::find_if(m_downloads.begin(), m_downloads.end(), has_index(piece)); + TORRENT_ASSERT(i != m_downloads.end()); - // ignore pieces found in the ignore list - if (std::find(ignore.begin(), ignore.end(), *i) != ignore.end()) continue; +// std::cout << "add_blocks_downloading(" << piece << ")" << std::endl; - TORRENT_ASSERT(m_piece_map[*i].priority(this) >= 0); - TORRENT_ASSERT(m_piece_map[*i].downloading == 0); - - int num_blocks_in_piece = blocks_in_piece(*i); + return add_blocks_downloading(*i, pieces + , interesting_blocks, backup_blocks, backup_blocks2 + , num_blocks, prefer_whole_pieces, peer, speed, options); + } - // pick a new piece - if (prefer_whole_pieces == 0) + int num_blocks_in_piece = blocks_in_piece(piece); + + // pick a new piece + if (prefer_whole_pieces == 0) + { + if (num_blocks_in_piece > num_blocks) + num_blocks_in_piece = num_blocks; + for (int j = 0; j < num_blocks_in_piece; ++j) + interesting_blocks.push_back(piece_block(piece, j)); + num_blocks -= num_blocks_in_piece; + } + else + { + int start, end; + boost::tie(start, end) = expand_piece(piece, prefer_whole_pieces, pieces); + for (int k = start; k < end; ++k) { - if (num_blocks_in_piece > num_blocks) - num_blocks_in_piece = num_blocks; + TORRENT_ASSERT(m_piece_map[k].priority(this) > 0); + num_blocks_in_piece = blocks_in_piece(k); for (int j = 0; j < num_blocks_in_piece; ++j) - interesting_blocks.push_back(piece_block(*i, j)); - num_blocks -= num_blocks_in_piece; - } - else - { - int start, end; - boost::tie(start, end) = expand_piece(*i, prefer_whole_pieces, pieces); - for (int k = start; k < end; ++k) { - TORRENT_ASSERT(m_piece_map[k].priority(this) > 0); - num_blocks_in_piece = blocks_in_piece(k); - for (int j = 0; j < num_blocks_in_piece; ++j) - { - interesting_blocks.push_back(piece_block(k, j)); - --num_blocks; - } + interesting_blocks.push_back(piece_block(k, j)); + --num_blocks; } } - if (num_blocks <= 0) - { -#ifndef NDEBUG - verify_pick(interesting_blocks, pieces); -#endif - return 0; - } } #ifndef NDEBUG verify_pick(interesting_blocks, pieces); #endif + if (num_blocks <= 0) return 0; return num_blocks; } - int piece_picker::add_blocks_downloading(bitfield const& pieces + int piece_picker::add_blocks_downloading(downloading_piece const& dp + , bitfield const& pieces , std::vector& interesting_blocks , std::vector& backup_blocks + , std::vector& backup_blocks2 , int num_blocks, int prefer_whole_pieces - , void* peer, piece_state_t speed, bool on_parole) const + , void* peer, piece_state_t speed, int options) const { - for (std::vector::const_iterator i = m_downloads.begin() - , end(m_downloads.end()); i != end; ++i) + if (!pieces[dp.index]) return num_blocks; + + int num_blocks_in_piece = blocks_in_piece(dp.index); + + // is true if all the other pieces that are currently + // requested from this piece are from the same + // peer as 'peer'. + bool exclusive; + bool exclusive_active; + boost::tie(exclusive, exclusive_active) + = requested_from(dp, num_blocks_in_piece, peer); + + // peers on parole are only allowed to pick blocks from + // pieces that only they have downloaded/requested from + if ((options & on_parole) && !exclusive) return num_blocks; + + // we prefer whole blocks, but there are other peers + // downloading from this piece, add it as backups + if (prefer_whole_pieces > 0 && !exclusive_active) { - if (!pieces[i->index]) continue; - - int num_blocks_in_piece = blocks_in_piece(i->index); - - // is true if all the other pieces that are currently - // requested from this piece are from the same - // peer as 'peer'. - bool exclusive; - bool exclusive_active; - boost::tie(exclusive, exclusive_active) - = requested_from(*i, num_blocks_in_piece, peer); - - // peers on parole are only allowed to pick blocks from - // pieces that only they have downloaded/requested from - if (on_parole && !exclusive) continue; - - if (prefer_whole_pieces > 0 && !exclusive_active) continue; - - // don't pick too many back-up blocks - if (i->state != none - && i->state != speed - && !exclusive_active - && int(backup_blocks.size()) >= num_blocks) - continue; + if (int(backup_blocks2.size()) >= num_blocks) + return num_blocks; for (int j = 0; j < num_blocks_in_piece; ++j) { // ignore completed blocks and already requested blocks - block_info const& info = i->info[j]; - if (info.state != block_info::state_none) - continue; - - TORRENT_ASSERT(i->info[j].state == block_info::state_none); - - // if the piece is fast and the peer is slow, or vice versa, - // add the block as a backup. - // override this behavior if all the other blocks - // have been requested from the same peer or - // if the state of the piece is none (the - // piece will in that case change state). - if (i->state != none && i->state != speed - && !exclusive_active) - { - backup_blocks.push_back(piece_block(i->index, j)); - continue; - } - - // this block is interesting (we don't have it - // yet). - interesting_blocks.push_back(piece_block(i->index, j)); - // we have found a block that's free to download - num_blocks--; - // if we prefer whole pieces, continue picking from this - // piece even though we have num_blocks - if (prefer_whole_pieces > 0) continue; - TORRENT_ASSERT(num_blocks >= 0); - if (num_blocks <= 0) break; + block_info const& info = dp.info[j]; + if (info.state != block_info::state_none) continue; + backup_blocks2.push_back(piece_block(dp.index, j)); } - if (num_blocks <= 0) break; + return num_blocks; + } + + for (int j = 0; j < num_blocks_in_piece; ++j) + { + // ignore completed blocks and already requested blocks + block_info const& info = dp.info[j]; + if (info.state != block_info::state_none) continue; + + TORRENT_ASSERT(dp.info[j].state == block_info::state_none); + + // if the piece is fast and the peer is slow, or vice versa, + // add the block as a backup. + // override this behavior if all the other blocks + // have been requested from the same peer or + // if the state of the piece is none (the + // piece will in that case change state). + if (dp.state != none && dp.state != speed + && !exclusive_active) + { + if (abs(dp.state - speed) == 1) + { + // don't pick too many back-up blocks + if (int(backup_blocks.size()) >= num_blocks) return num_blocks; + backup_blocks.push_back(piece_block(dp.index, j)); + } + else + { + // don't pick too many back-up blocks + if (int(backup_blocks2.size()) >= num_blocks) return num_blocks; + backup_blocks2.push_back(piece_block(dp.index, j)); + } + continue; + } + + // this block is interesting (we don't have it + // yet). + interesting_blocks.push_back(piece_block(dp.index, j)); + // we have found a block that's free to download + num_blocks--; + // if we prefer whole pieces, continue picking from this + // piece even though we have num_blocks + if (prefer_whole_pieces > 0) continue; + TORRENT_ASSERT(num_blocks >= 0); + if (num_blocks <= 0) return num_blocks; } TORRENT_ASSERT(num_blocks >= 0 || prefer_whole_pieces > 0); -#ifndef NDEBUG - verify_pick(interesting_blocks, pieces); - verify_pick(backup_blocks, pieces); -#endif - if (num_blocks <= 0) return 0; - if (on_parole) return num_blocks; - - int to_copy; - if (prefer_whole_pieces == 0) - to_copy = (std::min)(int(backup_blocks.size()), num_blocks); - else - to_copy = int(backup_blocks.size()); - - interesting_blocks.insert(interesting_blocks.end() - , backup_blocks.begin(), backup_blocks.begin() + to_copy); - num_blocks -= to_copy; - backup_blocks.clear(); - - if (num_blocks <= 0) return 0; - - if (prefer_whole_pieces > 0) - { - for (std::vector::const_iterator i = m_downloads.begin() - , end(m_downloads.end()); i != end; ++i) - { - if (!pieces[i->index]) continue; - int num_blocks_in_piece = blocks_in_piece(i->index); - bool exclusive; - bool exclusive_active; - boost::tie(exclusive, exclusive_active) - = requested_from(*i, num_blocks_in_piece, peer); - - if (exclusive_active) continue; - - for (int j = 0; j < num_blocks_in_piece; ++j) - { - block_info const& info = i->info[j]; - if (info.state != block_info::state_none) continue; - backup_blocks.push_back(piece_block(i->index, j)); - } - } - } + if (options & on_parole) return num_blocks; if (int(backup_blocks.size()) >= num_blocks) return num_blocks; - -#ifndef NDEBUG -// make sure that we at this point has added requests to all unrequested blocks -// in all downloading pieces - - for (std::vector::const_iterator i = m_downloads.begin() - , end(m_downloads.end()); i != end; ++i) - { - if (!pieces[i->index]) continue; - - int num_blocks_in_piece = blocks_in_piece(i->index); - for (int j = 0; j < num_blocks_in_piece; ++j) - { - block_info const& info = i->info[j]; - if (info.state != block_info::state_none) continue; - std::vector::iterator k = std::find( - interesting_blocks.begin(), interesting_blocks.end() - , piece_block(i->index, j)); - if (k != interesting_blocks.end()) continue; - - k = std::find(backup_blocks.begin() - , backup_blocks.end(), piece_block(i->index, j)); - if (k != backup_blocks.end()) continue; - - std::cerr << "interesting blocks:" << std::endl; - for (k = interesting_blocks.begin(); k != interesting_blocks.end(); ++k) - std::cerr << "(" << k->piece_index << ", " << k->block_index << ") "; - std::cerr << std::endl; - std::cerr << "backup blocks:" << std::endl; - for (k = backup_blocks.begin(); k != backup_blocks.end(); ++k) - std::cerr << "(" << k->piece_index << ", " << k->block_index << ") "; - std::cerr << std::endl; - std::cerr << "num_blocks: " << num_blocks << std::endl; - - for (std::vector::const_iterator l = m_downloads.begin() - , end(m_downloads.end()); l != end; ++l) - { - std::cerr << l->index << " : "; - int num_blocks_in_piece = blocks_in_piece(l->index); - for (int m = 0; m < num_blocks_in_piece; ++m) - std::cerr << l->info[m].state; - std::cerr << std::endl; - } - - TORRENT_ASSERT(false); - } - } -#endif - - for (std::vector::const_iterator i = m_downloads.begin() - , end(m_downloads.end()); i != end; ++i) - { - if (!pieces[i->index]) continue; - - int num_blocks_in_piece = blocks_in_piece(i->index); - - // fill in with blocks requested from other peers - // as backups - for (int j = 0; j < num_blocks_in_piece; ++j) - { - block_info const& info = i->info[j]; - if (info.state != block_info::state_requested - || info.peer == peer) - continue; - backup_blocks.push_back(piece_block(i->index, j)); - } - } #ifndef NDEBUG verify_pick(backup_blocks, pieces); #endif @@ -1696,6 +1924,7 @@ namespace libtorrent bool piece_picker::mark_as_downloading(piece_block block , void* peer, piece_state_t state) { + TORRENT_ASSERT(state != piece_picker::none); TORRENT_ASSERT(block.piece_index >= 0); TORRENT_ASSERT(block.block_index >= 0); TORRENT_ASSERT(block.piece_index < (int)m_piece_map.size()); @@ -1705,14 +1934,15 @@ namespace libtorrent piece_pos& p = m_piece_map[block.piece_index]; if (p.downloading == 0) { +#ifdef TORRENT_EXPENSIVE_INVARIANT_CHECKS TORRENT_PIECE_PICKER_INVARIANT_CHECK; +#endif int prio = p.priority(this); TORRENT_ASSERT(prio < int(m_priority_boundries.size()) - || m_sequential_download >= 0 || m_dirty); - TORRENT_ASSERT(prio > 0); + TORRENT_ASSERT(prio >= 0); p.downloading = 1; - if (prio >= 0 && m_sequential_download == -1 && !m_dirty) update(prio, p.index); + if (prio >= 0 && !m_dirty) update(prio, p.index); downloading_piece& dp = add_download_piece(); dp.state = state; @@ -1725,7 +1955,9 @@ namespace libtorrent } else { +#ifdef TORRENT_EXPENSIVE_INVARIANT_CHECKS TORRENT_PIECE_PICKER_INVARIANT_CHECK; +#endif std::vector::iterator i = std::find_if(m_downloads.begin(), m_downloads.end(), has_index(block.piece_index)); TORRENT_ASSERT(i != m_downloads.end()); @@ -1780,7 +2012,9 @@ namespace libtorrent void piece_picker::mark_as_writing(piece_block block, void* peer) { +#ifdef TORRENT_EXPENSIVE_INVARIANT_CHECKS TORRENT_PIECE_PICKER_INVARIANT_CHECK; +#endif TORRENT_ASSERT(block.piece_index >= 0); TORRENT_ASSERT(block.block_index >= 0); @@ -1801,7 +2035,9 @@ namespace libtorrent TORRENT_ASSERT(info.state != block_info::state_writing); ++i->writing; info.state = block_info::state_writing; - TORRENT_ASSERT(info.num_peers > 0); + + // all other requests for this block should have been + // cancelled now info.num_peers = 0; if (i->requested == 0) @@ -1822,22 +2058,13 @@ namespace libtorrent TORRENT_ASSERT(i != m_downloads.end()); block_info& info = i->info[block.block_index]; TORRENT_ASSERT(info.state == block_info::state_writing); + TORRENT_ASSERT(info.num_peers == 0); --i->writing; - if (info.num_peers > 0) - { - // there are other peers on this block - // turn it back into requested - ++i->requested; - info.state = block_info::state_requested; - } - else - { - info.state = block_info::state_none; - } + info.state = block_info::state_none; info.peer = 0; } - + void piece_picker::mark_as_finished(piece_block block, void* peer) { TORRENT_ASSERT(block.piece_index >= 0); @@ -1849,7 +2076,9 @@ namespace libtorrent if (p.downloading == 0) { +#ifdef TORRENT_EXPENSIVE_INVARIANT_CHECKS TORRENT_PIECE_PICKER_INVARIANT_CHECK; +#endif TORRENT_ASSERT(peer == 0); int prio = p.priority(this); @@ -1874,7 +2103,9 @@ namespace libtorrent } else { +#ifdef TORRENT_EXPENSIVE_INVARIANT_CHECKS TORRENT_PIECE_PICKER_INVARIANT_CHECK; +#endif std::vector::iterator i = std::find_if(m_downloads.begin(), m_downloads.end(), has_index(block.piece_index)); @@ -1893,6 +2124,7 @@ namespace libtorrent } else { + TORRENT_ASSERT(info.state == block_info::state_none); info.state = block_info::state_finished; sort_piece(i); } @@ -1954,23 +2186,31 @@ namespace libtorrent block_info& info = i->info[block.block_index]; - if (i->info[block.block_index].state != block_info::state_requested) + TORRENT_ASSERT(info.state != block_info::state_none); + + if (info.state == block_info::state_finished + || info.state == block_info::state_none + || info.state == block_info::state_writing) return; - if (info.num_peers > 0) --info.num_peers; + if (info.state == block_info::state_requested) + { + TORRENT_ASSERT(info.num_peers > 0); + if (info.num_peers > 0) --info.num_peers; - TORRENT_ASSERT(block.block_index < blocks_in_piece(block.piece_index)); + TORRENT_ASSERT(block.block_index < blocks_in_piece(block.piece_index)); - // if there are other peers, leave the block requested - if (info.num_peers > 0) return; + // if there are other peers, leave the block requested + if (info.num_peers > 0) return; - // clear the downloader of this block - info.peer = 0; + // clear the downloader of this block + info.peer = 0; + + // clear this block as being downloaded + info.state = block_info::state_none; + --i->requested; + } - // clear this block as being downloaded - info.state = block_info::state_none; - --i->requested; - // if there are no other blocks in this piece // that's being downloaded, remove it from the list if (i->requested + i->finished + i->writing == 0) @@ -1981,7 +2221,7 @@ namespace libtorrent TORRENT_ASSERT(prev_prio < int(m_priority_boundries.size()) || m_dirty); p.downloading = 0; - if (m_sequential_download == -1 && !m_dirty) + if (!m_dirty) { int prio = p.priority(this); if (prev_prio == -1 && prio >= 0) add(block.piece_index); diff --git a/src/policy.cpp b/src/policy.cpp index f51b563cd..b98ce6363 100644 --- a/src/policy.cpp +++ b/src/policy.cpp @@ -200,8 +200,6 @@ namespace libtorrent int prefer_whole_pieces = c.prefer_whole_pieces(); - bool rarest_first = t.num_have() >= t.settings().initial_picker_threshold; - if (prefer_whole_pieces == 0) { prefer_whole_pieces = c.statistics().download_payload_rate() @@ -250,7 +248,7 @@ namespace libtorrent p.pick_pieces(mask, interesting_pieces , num_requests, prefer_whole_pieces, c.peer_info_struct() - , state, rarest_first, c.on_parole(), suggested); + , state, c.picker_options(), suggested); } else { @@ -264,7 +262,7 @@ namespace libtorrent // then use this mode. p.pick_pieces(bits, interesting_pieces , num_requests, prefer_whole_pieces, c.peer_info_struct() - , state, rarest_first, c.on_parole(), suggested); + , state, c.picker_options(), suggested); } #ifdef TORRENT_VERBOSE_LOGGING @@ -303,10 +301,6 @@ namespace libtorrent if (busy_pieces.empty() || num_requests <= 0) { - // in this case, we could not find any blocks - // that was free. If we couldn't find any busy - // blocks as well, we cannot download anything - // more from this peer. return; } diff --git a/src/torrent.cpp b/src/torrent.cpp index 6b8cce969..b5c43d918 100644 --- a/src/torrent.cpp +++ b/src/torrent.cpp @@ -754,10 +754,14 @@ namespace libtorrent m_progress = j.piece / float(torrent_file().num_pieces()); + m_picker->check_invariant(); + TORRENT_ASSERT(m_picker); if (j.offset >= 0 && !m_picker->have_piece(j.offset)) m_picker->we_have(j.offset); + m_picker->check_invariant(); + // we're not done checking yet // this handler will be called repeatedly until // we're done, or encounter a failure @@ -3298,9 +3302,6 @@ namespace libtorrent if (!is_seed()) { - if (m_sequential_download) - picker().sequential_download(m_sequential_download); - // if we just finished checking and we're not a seed, we are // likely to be unpaused if (m_ses.m_auto_manage_time_scaler > 1) @@ -3549,13 +3550,7 @@ namespace libtorrent #endif void torrent::set_sequential_download(bool sd) - { - m_sequential_download = sd; - if (has_picker()) - { - picker().sequential_download(sd); - } - } + { m_sequential_download = sd; } void torrent::set_queue_position(int p) { diff --git a/test/test_piece_picker.cpp b/test/test_piece_picker.cpp index b90fa2237..2ce27223c 100644 --- a/test/test_piece_picker.cpp +++ b/test/test_piece_picker.cpp @@ -71,6 +71,14 @@ boost::shared_ptr setup_picker( boost::shared_ptr p(new piece_picker); p->init(blocks_per_piece, num_pieces * blocks_per_piece); + for (int i = 0; i < num_pieces; ++i) + { + const int avail = availability[i] - '0'; + assert(avail >= 0); + + for (int j = 0; j < avail; ++j) p->inc_refcount(i); + } + bitfield have = string2vec(have_str); for (int i = 0; i < num_pieces; ++i) @@ -86,25 +94,24 @@ boost::shared_ptr setup_picker( blocks = partial[i] - 'a' + 10; int counter = 0; - if (blocks & 1) + for (int j = 0; j < 4; ++j) { + TEST_CHECK(!p->is_finished(piece_block(i, j))); + if ((blocks & (1 << j)) == 0) continue; ++counter; - p->mark_as_finished(piece_block(i, 0), 0); - } - if (blocks & 2) - { - ++counter; - p->mark_as_finished(piece_block(i, 1), 0); - } - if (blocks & 4) - { - ++counter; - p->mark_as_finished(piece_block(i, 2), 0); - } - if (blocks & 8) - { - ++counter; - p->mark_as_finished(piece_block(i, 3), 0); + bool ret = p->mark_as_downloading(piece_block(i, j), 0, piece_picker::slow); + TEST_CHECK(ret == true); + TEST_CHECK(p->is_requested(piece_block(i, j)) == bool(blocks & (1 << j))); + p->mark_as_writing(piece_block(i, j), 0); + TEST_CHECK(!p->is_finished(piece_block(i, j))); + // trying to mark a block as requested after it has been completed + // should fail (return false) + ret = p->mark_as_downloading(piece_block(i, j), 0, piece_picker::slow); + TEST_CHECK(ret == false); + p->mark_as_finished(piece_block(i, j), 0); + + TEST_CHECK(p->is_downloaded(piece_block(i, j)) == bool(blocks & (1 << j))); + TEST_CHECK(p->is_finished(piece_block(i, j)) == bool(blocks & (1 << j))); } piece_picker::downloading_piece st; @@ -114,6 +121,9 @@ boost::shared_ptr setup_picker( TEST_CHECK(st.index == i); TEST_CHECK(st.finished == counter); + TEST_CHECK(st.finished + st.requested + st.writing == counter); + + TEST_CHECK(p->is_piece_finished(i) == (counter == 4)); } for (int i = 0; i < num_pieces; ++i) @@ -134,14 +144,6 @@ boost::shared_ptr setup_picker( TEST_CHECK(p->is_finished(piece_block(i, j))); } - for (int i = 0; i < num_pieces; ++i) - { - const int avail = availability[i] - '0'; - assert(avail >= 0); - - for (int j = 0; j < avail; ++j) p->inc_refcount(i); - } - std::vector availability_vec; p->get_availability(availability_vec); for (int i = 0; i < num_pieces; ++i) @@ -158,20 +160,25 @@ boost::shared_ptr setup_picker( } bool verify_pick(boost::shared_ptr p - , std::vector const& picked) + , std::vector const& picked, bool allow_multi_blocks = false) { #ifndef NDEBUG p->check_invariant(); #endif - for (std::vector::const_iterator i = picked.begin() - , end(picked.end()); i != end; ++i) + if (!allow_multi_blocks) { - if (p->num_peers(*i) > 0) return false; + for (std::vector::const_iterator i = picked.begin() + , end(picked.end()); i != end; ++i) + { + if (p->num_peers(*i) > 0) return false; + } } // make sure there are no duplicated std::set blocks; - std::copy(picked.begin(), picked.end(), std::insert_iterator >(blocks, blocks.end())); + std::copy(picked.begin(), picked.end() + , std::insert_iterator >(blocks, blocks.end())); + std::cerr << " verify: " << picked.size() << " " << blocks.size() << std::endl; return picked.size() == blocks.size(); } @@ -189,60 +196,182 @@ void print_title(char const* name) std::cerr << "==== " << name << " ====\n"; } -int test_pick(boost::shared_ptr const& p) +std::vector pick_pieces(boost::shared_ptr const& p, char const* availability + , int num_blocks, int prefer_whole_pieces, void* peer_struct, piece_picker::piece_state_t state + , int options, std::vector const& suggested_pieces) { std::vector picked; - const std::vector empty_vector; - p->pick_pieces(string2vec("*******"), picked, 1, false, 0, piece_picker::fast, true, false, empty_vector); + p->pick_pieces(string2vec(availability), picked, num_blocks, prefer_whole_pieces, peer_struct + , state, options, suggested_pieces); print_pick(picked); TEST_CHECK(verify_pick(p, picked)); - TEST_CHECK(int(picked.size()) == 1); + return picked; +} + +int test_pick(boost::shared_ptr const& p, int options = piece_picker::rarest_first) +{ + const std::vector empty_vector; + std::vector picked = pick_pieces(p, "*******", 1, 0, 0 + , piece_picker::fast, options, empty_vector); + if (picked.empty()) return -1; return picked[0].piece_index; } int test_main() { + int tmp1; + int tmp2; + int tmp3; tcp::endpoint endp; + piece_picker::downloading_piece st; policy::peer peer_struct(endp, policy::peer::connectable, 0); std::vector picked; boost::shared_ptr p; const std::vector empty_vector; + int options = piece_picker::rarest_first; + +// ======================================================== + + // test abort_download + print_title("test abort_download"); + p = setup_picker("1111111", " ", "7110000", ""); + picked = pick_pieces(p, "*******", blocks_per_piece, 0, 0, piece_picker::fast + , options, empty_vector); + TEST_CHECK(p->is_requested(piece_block(0, 0)) == false); + TEST_CHECK(std::find(picked.begin(), picked.end(), piece_block(0,0)) != picked.end()); + + p->abort_download(piece_block(piece_block(0,0))); + picked = pick_pieces(p, "*******", blocks_per_piece, 0, 0, piece_picker::fast + , options, empty_vector); + TEST_CHECK(p->is_requested(piece_block(0, 0)) == false); + TEST_CHECK(std::find(picked.begin(), picked.end(), piece_block(0,0)) != picked.end()); + + p->mark_as_downloading(piece_block(0,0), &tmp1, piece_picker::fast); + picked = pick_pieces(p, "*******", blocks_per_piece, 0, 0, piece_picker::fast + , options, empty_vector); + TEST_CHECK(p->is_requested(piece_block(0, 0)) == true); + TEST_CHECK(std::find(picked.begin(), picked.end(), piece_block(0,0)) == picked.end()); + + p->abort_download(piece_block(0,0)); + picked = pick_pieces(p, "*******", blocks_per_piece, 0, 0, piece_picker::fast + , options, empty_vector); + TEST_CHECK(p->is_requested(piece_block(0, 0)) == false); + TEST_CHECK(std::find(picked.begin(), picked.end(), piece_block(0,0)) != picked.end()); + + p->mark_as_downloading(piece_block(0,0), &tmp1, piece_picker::fast); + p->mark_as_downloading(piece_block(0,1), &tmp1, piece_picker::fast); + p->abort_download(piece_block(0,0)); + picked = pick_pieces(p, "*******", blocks_per_piece, 0, 0, piece_picker::fast + , options, empty_vector); + TEST_CHECK(p->is_requested(piece_block(0, 0)) == false); + TEST_CHECK(std::find(picked.begin(), picked.end(), piece_block(0,0)) != picked.end()); + + p->mark_as_downloading(piece_block(0,0), &tmp1, piece_picker::fast); + p->mark_as_writing(piece_block(0,0), &tmp1); + p->write_failed(piece_block(0,0)); + picked = pick_pieces(p, "*******", blocks_per_piece, 0, 0, piece_picker::fast + , options, empty_vector); + TEST_CHECK(p->is_requested(piece_block(0, 0)) == false); + TEST_CHECK(std::find(picked.begin(), picked.end(), piece_block(0,0)) != picked.end()); + + p->mark_as_downloading(piece_block(0,0), &tmp1, piece_picker::fast); + p->mark_as_writing(piece_block(0,0), &tmp1); + p->mark_as_finished(piece_block(0,0), &tmp1); + p->abort_download(piece_block(0,0)); + picked = pick_pieces(p, "*******", blocks_per_piece, 0, 0, piece_picker::fast + , options, empty_vector); + TEST_CHECK(p->is_requested(piece_block(0, 0)) == false); + TEST_CHECK(std::find(picked.begin(), picked.end(), piece_block(0,0)) == picked.end()); + + p = setup_picker("1111111", " ", "7110000", ""); + p->mark_as_downloading(piece_block(0,0), &tmp1, piece_picker::fast); + p->mark_as_finished(piece_block(0,1), 0); + p->piece_info(0, st); + TEST_CHECK(st.requested == 1); + TEST_CHECK(st.finished == 1); + TEST_CHECK(st.state == piece_picker::fast); + p->abort_download(piece_block(0,0)); + p->piece_info(0, st); + TEST_CHECK(st.requested == 0); + TEST_CHECK(st.finished == 1); + TEST_CHECK(st.state == piece_picker::none); + picked = pick_pieces(p, "*******", blocks_per_piece, 0, 0, piece_picker::fast + , options, empty_vector); + TEST_CHECK(p->is_requested(piece_block(0, 0)) == false); + TEST_CHECK(std::find(picked.begin(), picked.end(), piece_block(0,0)) != picked.end()); + +// ======================================================== // make sure the block that is picked is from piece 1, since it // it is the piece with the lowest availability print_title("test pick lowest availability"); p = setup_picker("2223333", "* * * ", "", ""); - picked.clear(); - p->pick_pieces(string2vec("*******"), picked, 1, false, 0, piece_picker::fast, true, false, empty_vector); - TEST_CHECK(verify_pick(p, picked)); - TEST_CHECK(int(picked.size()) > 0); - TEST_CHECK(picked.front().piece_index == 1); + TEST_CHECK(test_pick(p) == 1); +// ======================================================== + + // make sure pieces with equal priority and availability + // are picked at random + print_title("test random pick at same priority"); + std::map random_prio_pieces; + for (int i = 0; i < 100; ++i) + { + p = setup_picker("1111112", " ", "", ""); + ++random_prio_pieces[test_pick(p)]; + } + TEST_CHECK(random_prio_pieces.size() == 6); + for (std::map::iterator i = random_prio_pieces.begin() + , end(random_prio_pieces.end()); i != end; ++i) + std::cout << i->first << ": " << i->second << " "; + std::cout << std::endl; + // ======================================================== // make sure the block that is picked is from piece 5, since it // has the highest priority among the available pieces print_title("test pick highest priority"); - p = setup_picker("1111111", "* * * ", "1111122", ""); - picked.clear(); - p->pick_pieces(string2vec("****** "), picked, 1, false, 0, piece_picker::fast, true, false, empty_vector); - TEST_CHECK(verify_pick(p, picked)); - TEST_CHECK(int(picked.size()) > 0); - TEST_CHECK(picked.front().piece_index == 5); + p = setup_picker("1111111", "* * * ", "1111121", ""); + TEST_CHECK(test_pick(p) == 5); + +// ======================================================== + + print_title("test reverse rarest first"); + p = setup_picker("4179253", " ", "", ""); + picked = pick_pieces(p, "*******", 7 * blocks_per_piece, 0, &peer_struct, piece_picker::fast + , piece_picker::rarest_first | piece_picker::reverse, empty_vector); + int expected_common_pieces[] = {3, 2, 5, 0, 6, 4, 1}; + for (int i = 0; i < int(picked.size()); ++i) + TEST_CHECK(picked[i] == piece_block(expected_common_pieces[i / blocks_per_piece], i % blocks_per_piece)); + + // piece 3 should be prioritized since it's a partial + p = setup_picker("1122111", " ", "3333333", " 1 "); + TEST_CHECK(test_pick(p, piece_picker::rarest_first | piece_picker::reverse) == 3); // ======================================================== // make sure the 4 blocks are picked from the same piece if - // whole pieces are preferred. The only whole piece is 1. + // whole pieces are preferred. Priority and availability is more + // important. Piece 1 has the lowest availability even though + // it is not a whole piece print_title("test pick whole pieces"); - p = setup_picker("1111111", " ", "1111111", "1023460"); - picked.clear(); - p->pick_pieces(string2vec("****** "), picked, 1, 1, &peer_struct, piece_picker::fast, true, true, empty_vector); - TEST_CHECK(verify_pick(p, picked)); - TEST_CHECK(int(picked.size()) >= blocks_per_piece); + p = setup_picker("2212222", " ", "1111111", "1023460"); + picked = pick_pieces(p, "****** ", 1, 1, &peer_struct, piece_picker::fast, options, empty_vector); + TEST_CHECK(int(picked.size()) == 3); for (int i = 0; i < blocks_per_piece && i < int(picked.size()); ++i) - TEST_CHECK(picked[i].piece_index == 1); + TEST_CHECK(picked[i].piece_index == 2); + + p = setup_picker("1111111", " ", "1111111", ""); + picked = pick_pieces(p, "****** ", 1, 1, &peer_struct, piece_picker::fast, options, empty_vector); + TEST_CHECK(int(picked.size()) == blocks_per_piece); + for (int i = 0; i < blocks_per_piece && i < int(picked.size()); ++i) + TEST_CHECK(picked[i].block_index == i); + + p = setup_picker("2221222", " ", "", ""); + picked = pick_pieces(p, "*******", 1, 7, &peer_struct, piece_picker::fast, options, empty_vector); + TEST_CHECK(int(picked.size()) == 7 * blocks_per_piece); + for (int i = 0; i < int(picked.size()); ++i) + TEST_CHECK(picked[i] == piece_block(i / blocks_per_piece, i % blocks_per_piece)); // ======================================================== @@ -259,22 +388,23 @@ int test_main() // make sure filtered pieces are ignored print_title("test filtered pieces"); p = setup_picker("1111111", " ", "0010000", ""); - picked.clear(); - p->pick_pieces(string2vec("*** ** "), picked, 1, false, 0, piece_picker::fast, true, false, empty_vector); - TEST_CHECK(verify_pick(p, picked)); - TEST_CHECK(int(picked.size()) > 0); - TEST_CHECK(picked.front().piece_index == 2); + TEST_CHECK(test_pick(p, piece_picker::rarest_first) == 2); + TEST_CHECK(test_pick(p, piece_picker::rarest_first | piece_picker::reverse) == 2); + TEST_CHECK(test_pick(p, piece_picker::sequential) == 2); + TEST_CHECK(test_pick(p, piece_picker::sequential | piece_picker::reverse) == 2); // ======================================================== // make sure we_dont_have works print_title("test we_dont_have"); p = setup_picker("1111111", "*******", "0100000", ""); - picked.clear(); + TEST_CHECK(p->have_piece(1)); + TEST_CHECK(p->have_piece(2)); p->we_dont_have(1); p->we_dont_have(2); - p->pick_pieces(string2vec("*** ** "), picked, 1, false, 0, piece_picker::fast, true, false, empty_vector); - TEST_CHECK(verify_pick(p, picked)); + TEST_CHECK(!p->have_piece(1)); + TEST_CHECK(!p->have_piece(2)); + picked = pick_pieces(p, "*** ** ", 1, 0, 0, piece_picker::fast, options, empty_vector); TEST_CHECK(int(picked.size()) > 0); TEST_CHECK(picked.front().piece_index == 1); @@ -309,76 +439,122 @@ int test_main() // make sure requested blocks aren't picked print_title("test don't pick requested blocks"); - p = setup_picker("1234567", " ", "", ""); - picked.clear(); - p->pick_pieces(string2vec("*******"), picked, 1, false, 0, piece_picker::fast, true, false, empty_vector); - TEST_CHECK(verify_pick(p, picked)); + p = setup_picker("1111111", " ", "", ""); + picked = pick_pieces(p, "*******", 1, 0, 0, piece_picker::fast, options, empty_vector); TEST_CHECK(int(picked.size()) > 0); - TEST_CHECK(picked.front().piece_index == 0); piece_block first = picked.front(); p->mark_as_downloading(picked.front(), &peer_struct, piece_picker::fast); TEST_CHECK(p->num_peers(picked.front()) == 1); - picked.clear(); - p->pick_pieces(string2vec("*******"), picked, 1, false, 0, piece_picker::fast, true, false, empty_vector); - TEST_CHECK(verify_pick(p, picked)); + picked = pick_pieces(p, "*******", 1, 0, 0, piece_picker::fast, options, empty_vector); TEST_CHECK(int(picked.size()) > 0); TEST_CHECK(picked.front() != first); - TEST_CHECK(picked.front().piece_index == 0); // ======================================================== -/* - // test sequenced download + + // make sure downloading pieces have higher priority + print_title("test downloading piece priority"); + p = setup_picker("1111111", " ", "", ""); + picked = pick_pieces(p, "*******", 1, 0, 0, piece_picker::fast, options, empty_vector); + TEST_CHECK(int(picked.size()) > 0); + first = picked.front(); + p->mark_as_downloading(picked.front(), &peer_struct, piece_picker::fast); + TEST_CHECK(p->num_peers(picked.front()) == 1); + picked = pick_pieces(p, "*******", 1, 0, 0, piece_picker::fast, options, empty_vector); + TEST_CHECK(int(picked.size()) > 0); + TEST_CHECK(picked.front() != first); + TEST_CHECK(picked.front().piece_index == first.piece_index); + +// ======================================================== + + // make sure downloading pieces closer to completion have higher priority + // piece 3 has only 1 block from being completed, and should be picked + print_title("test downloading piece order"); + p = setup_picker("1111111", " ", "", "013700f"); + picked = pick_pieces(p, "*******", 1, 0, 0, piece_picker::fast + , options | piece_picker::prioritize_partials, empty_vector); + TEST_CHECK(int(picked.size()) > 0); + TEST_CHECK(picked.front() == piece_block(3, 3)); + +// ======================================================== + + // test sequential download + print_title("test sequential download"); p = setup_picker("7654321", " ", "", ""); - picked.clear(); - p->set_sequenced_download_threshold(3); - p->pick_pieces(string2vec("***** "), picked, 5 * blocks_per_piece, false, 0, piece_picker::fast, true, false, empty_vector); - print_pick(picked); - TEST_CHECK(verify_pick(p, picked)); - TEST_CHECK(int(picked.size()) == 5 * blocks_per_piece); - for (int i = 0; i < 5 * blocks_per_piece && i < int(picked.size()); ++i) - TEST_CHECK(picked[i].piece_index == i / blocks_per_piece); + picked = pick_pieces(p, "*******", 7 * blocks_per_piece, 0, 0, piece_picker::fast + , piece_picker::sequential, empty_vector); + TEST_CHECK(int(picked.size()) == 7 * blocks_per_piece); + for (int i = 0; i < int(picked.size()); ++i) + TEST_CHECK(picked[i] == piece_block(i / blocks_per_piece, i % blocks_per_piece)); - picked.clear(); - p->set_sequenced_download_threshold(4); - p->pick_pieces(string2vec("**** "), picked, 5 * blocks_per_piece, false, 0, piece_picker::fast, true, false, empty_vector); - print_pick(picked); - TEST_CHECK(verify_pick(p, picked)); - TEST_CHECK(int(picked.size()) == 4 * blocks_per_piece); - for (int i = 0; i < 4 * blocks_per_piece && i < int(picked.size()); ++i) - TEST_CHECK(picked[i].piece_index == i / blocks_per_piece); +// ======================================================== - picked.clear(); - p->set_sequenced_download_threshold(2); - p->pick_pieces(string2vec("****** "), picked, 6 * blocks_per_piece, false, 0, piece_picker::fast, true, false, empty_vector); - print_pick(picked); - TEST_CHECK(verify_pick(p, picked)); - TEST_CHECK(int(picked.size()) == 6 * blocks_per_piece); - for (int i = 0; i < 6 * blocks_per_piece && i < int(picked.size()); ++i) - TEST_CHECK(picked[i].piece_index == i / blocks_per_piece); - - picked.clear(); + // test reverse sequential download + print_title("test reverse sequential download"); + p = setup_picker("7654321", " ", "", ""); + picked = pick_pieces(p, "*******", 7 * blocks_per_piece, 0, 0, piece_picker::fast + , piece_picker::sequential | piece_picker::reverse, empty_vector); + TEST_CHECK(int(picked.size()) == 7 * blocks_per_piece); + for (int i = 0; i < int(picked.size()); ++i) + TEST_CHECK(picked[i] == piece_block(6 - (i / blocks_per_piece), i % blocks_per_piece)); + +// ======================================================== + + // test cursors + print_title("test cursors"); + p = setup_picker("7654321", " ", "", ""); + TEST_CHECK(p->cursor() == 0); + TEST_CHECK(p->reverse_cursor() == 7); + p->we_have(1); + TEST_CHECK(p->cursor() == 0); + TEST_CHECK(p->reverse_cursor() == 7); + p->we_have(0); + TEST_CHECK(p->cursor() == 2); + TEST_CHECK(p->reverse_cursor() == 7); + p->we_have(5); + TEST_CHECK(p->cursor() == 2); + TEST_CHECK(p->reverse_cursor() == 7); + p->we_have(6); + TEST_CHECK(p->cursor() == 2); + TEST_CHECK(p->reverse_cursor() == 5); + p->we_have(4); + p->we_have(3); + p->we_have(2); + TEST_CHECK(p->cursor() == 7); + TEST_CHECK(p->reverse_cursor() == 0); + p->we_dont_have(3); + TEST_CHECK(p->cursor() == 3); + TEST_CHECK(p->reverse_cursor() == 4); + + p = setup_picker("7654321", " ", "", ""); + TEST_CHECK(p->cursor() == 0); + TEST_CHECK(p->reverse_cursor() == 7); + p->set_piece_priority(1, 0); + TEST_CHECK(p->cursor() == 0); + TEST_CHECK(p->reverse_cursor() == 7); p->set_piece_priority(0, 0); - p->pick_pieces(string2vec("****** "), picked, 6 * blocks_per_piece, false, 0, piece_picker::fast, true, false, empty_vector); - print_pick(picked); - TEST_CHECK(verify_pick(p, picked)); - TEST_CHECK(int(picked.size()) == 5 * blocks_per_piece); - for (int i = 0; i < 5 * blocks_per_piece && i < int(picked.size()); ++i) - TEST_CHECK(picked[i].piece_index == i / blocks_per_piece + 1); + TEST_CHECK(p->cursor() == 2); + TEST_CHECK(p->reverse_cursor() == 7); + p->set_piece_priority(5, 0); + TEST_CHECK(p->cursor() == 2); + TEST_CHECK(p->reverse_cursor() == 7); + p->set_piece_priority(6, 0); + TEST_CHECK(p->cursor() == 2); + TEST_CHECK(p->reverse_cursor() == 5); + p->set_piece_priority(4, 0); + p->set_piece_priority(3, 0); + p->set_piece_priority(2, 0); + TEST_CHECK(p->cursor() == 7); + TEST_CHECK(p->reverse_cursor() == 0); + p->set_piece_priority(3, 1); + TEST_CHECK(p->cursor() == 3); + TEST_CHECK(p->reverse_cursor() == 4); + - picked.clear(); - p->set_piece_priority(0, 1); - p->pick_pieces(string2vec("****** "), picked, 6 * blocks_per_piece, false, 0, piece_picker::fast, true, false, empty_vector); - print_pick(picked); - TEST_CHECK(verify_pick(p, picked)); - TEST_CHECK(int(picked.size()) == 6 * blocks_per_piece); - for (int i = 0; i < 6 * blocks_per_piece && i < int(picked.size()); ++i) - TEST_CHECK(picked[i].piece_index == i / blocks_per_piece); -*/ // ======================================================== // test piece priorities print_title("test piece priorities"); - p = setup_picker("5555555", " ", "3214576", ""); + p = setup_picker("5555555", " ", "7654321", ""); TEST_CHECK(p->num_filtered() == 0); TEST_CHECK(p->num_have_filtered() == 0); p->set_piece_priority(0, 0); @@ -389,25 +565,34 @@ int test_main() TEST_CHECK(p->num_filtered() == 0); TEST_CHECK(p->num_have_filtered() == 1); - picked.clear(); - p->pick_pieces(string2vec("*******"), picked, 6 * blocks_per_piece, false, 0, piece_picker::fast, true, false, empty_vector); - print_pick(picked); - TEST_CHECK(verify_pick(p, picked)); - TEST_CHECK(int(picked.size()) == 6 * blocks_per_piece); - TEST_CHECK(picked[0 * blocks_per_piece].piece_index == 5); - // priority 5 and 6 is currently the same - TEST_CHECK(picked[1 * blocks_per_piece].piece_index == 6 || picked[1 * blocks_per_piece].piece_index == 4); - TEST_CHECK(picked[2 * blocks_per_piece].piece_index == 6 || picked[2 * blocks_per_piece].piece_index == 4); - TEST_CHECK(picked[3 * blocks_per_piece].piece_index == 3); - TEST_CHECK(picked[4 * blocks_per_piece].piece_index == 1); - TEST_CHECK(picked[5 * blocks_per_piece].piece_index == 2); + p->we_dont_have(0); + p->set_piece_priority(0, 7); + + picked = pick_pieces(p, "*******", 7 * blocks_per_piece, 0, 0 + , piece_picker::fast, options, empty_vector); + TEST_CHECK(int(picked.size()) == 7 * blocks_per_piece); + + for (int i = 0; i < int(picked.size()); ++i) + TEST_CHECK(picked[i] == piece_block(i / blocks_per_piece, i % blocks_per_piece)); + + // test changing priority on a piece we have + p->we_have(0); + p->set_piece_priority(0, 0); + p->set_piece_priority(0, 1); + p->set_piece_priority(0, 0); std::vector prios; p->piece_priorities(prios); TEST_CHECK(prios.size() == 7); - int prio_comp[] = {0, 2, 1, 4, 5, 7, 6}; + int prio_comp[] = {0, 6, 5, 4, 3, 2, 1}; TEST_CHECK(std::equal(prios.begin(), prios.end(), prio_comp)); + std::vector filter; + p->filtered_pieces(filter); + TEST_CHECK(prios.size() == 7); + bool filter_comp[] = {true, false, false, false, false, false, false}; + TEST_CHECK(std::equal(filter.begin(), filter.end(), filter_comp)); + // ======================================================== // test restore_piece @@ -418,18 +603,12 @@ int test_main() p->mark_as_finished(piece_block(0,2), 0); p->mark_as_finished(piece_block(0,3), 0); - picked.clear(); - p->pick_pieces(string2vec("*******"), picked, 1, false, 0, piece_picker::fast, true, false, empty_vector); - print_pick(picked); - TEST_CHECK(verify_pick(p, picked)); + picked = pick_pieces(p, "*******", 1, 0, 0, piece_picker::fast, options, empty_vector); TEST_CHECK(int(picked.size()) >= 1); TEST_CHECK(picked.front().piece_index == 1); p->restore_piece(0); - picked.clear(); - p->pick_pieces(string2vec("*******"), picked, 1, false, 0, piece_picker::fast, true, false, empty_vector); - print_pick(picked); - TEST_CHECK(verify_pick(p, picked)); + picked = pick_pieces(p, "*******", 1, 0, 0, piece_picker::fast, options, empty_vector); TEST_CHECK(int(picked.size()) >= 1); TEST_CHECK(picked.front().piece_index == 0); @@ -439,46 +618,128 @@ int test_main() p->mark_as_finished(piece_block(0,3), 0); p->set_piece_priority(0, 0); - picked.clear(); - p->pick_pieces(string2vec("*******"), picked, 1, false, 0, piece_picker::fast, true, false, empty_vector); - print_pick(picked); - TEST_CHECK(verify_pick(p, picked)); + picked = pick_pieces(p, "*******", 1, 0, 0, piece_picker::fast, options, empty_vector); TEST_CHECK(int(picked.size()) >= 1); TEST_CHECK(picked.front().piece_index == 1); p->restore_piece(0); - picked.clear(); - p->pick_pieces(string2vec("*******"), picked, 1, false, 0, piece_picker::fast, true, false, empty_vector); - print_pick(picked); - TEST_CHECK(verify_pick(p, picked)); + picked = pick_pieces(p, "*******", 1, 0, 0, piece_picker::fast, options, empty_vector); TEST_CHECK(int(picked.size()) >= 1); TEST_CHECK(picked.front().piece_index == 1); p->set_piece_priority(0, 1); - picked.clear(); - p->pick_pieces(string2vec("*******"), picked, 1, false, 0, piece_picker::fast, true, false, empty_vector); - print_pick(picked); - TEST_CHECK(verify_pick(p, picked)); + picked = pick_pieces(p, "*******", 1, 0, 0, piece_picker::fast, options, empty_vector); TEST_CHECK(int(picked.size()) >= 1); TEST_CHECK(picked.front().piece_index == 0); // ======================================================== - // test non-rarest-first mode - print_title("test not rarest first"); - p = setup_picker("1234567", "* * * ", "1111122", ""); - picked.clear(); - p->pick_pieces(string2vec("****** "), picked, 5 * blocks_per_piece, false, 0, piece_picker::fast, false, false, empty_vector); - print_pick(picked); - TEST_CHECK(verify_pick(p, picked)); - TEST_CHECK(int(picked.size()) == 3 * blocks_per_piece); + // test random mode + print_title("test random pick"); + p = setup_picker("1234567", " ", "1111122", ""); + std::set random_pieces; + for (int i = 0; i < 100; ++i) + random_pieces.insert(test_pick(p, 0)); + TEST_CHECK(random_pieces.size() == 7); - for (int i = 0; i < 4 * blocks_per_piece && i < int(picked.size()); ++i) + random_pieces.clear(); + for (int i = 0; i < 7; ++i) { - TEST_CHECK(picked[i].piece_index != 0); - TEST_CHECK(picked[i].piece_index != 2); - TEST_CHECK(picked[i].piece_index != 4); + int piece = test_pick(p, 0); + p->we_have(piece); + random_pieces.insert(piece); } + TEST_CHECK(random_pieces.size() == 7); + +// ======================================================== + + // make sure that blocks from a slow piece are picked + // by a fast peer if there are no other options + print_title("test downloading piece affinity"); + p = setup_picker("1111111", " ", "", ""); + p->mark_as_downloading(piece_block(2,2), &tmp1, piece_picker::slow); + picked = pick_pieces(p, "*******", 7 * blocks_per_piece - 1, 0, &tmp2 + , piece_picker::fast, options, empty_vector); + TEST_CHECK(picked.size() == 7 * blocks_per_piece - 1); + TEST_CHECK(std::find(picked.begin(), picked.end(), piece_block(2,2)) == picked.end()); + // piece 2 sould be the last one (least matching piece to pick) + TEST_CHECK(picked[7 * blocks_per_piece - 2].piece_index == 2); + TEST_CHECK(picked[7 * blocks_per_piece - 3].piece_index == 2); + TEST_CHECK(picked[7 * blocks_per_piece - 4].piece_index == 2); + + // test the affinity of pieces with the same speed state + p = setup_picker("1111111", " ", "", ""); + p->mark_as_downloading(piece_block(3,2), &tmp1, piece_picker::slow); + p->mark_as_downloading(piece_block(2,2), &tmp1, piece_picker::medium); + p->mark_as_downloading(piece_block(4,2), &tmp1, piece_picker::fast); + picked = pick_pieces(p, "*******", 2 * blocks_per_piece, 0, 0 + , piece_picker::fast, piece_picker::prioritize_partials, empty_vector); + TEST_CHECK(picked.size() == 2 * blocks_per_piece); + TEST_CHECK(picked[0].piece_index == 4); + TEST_CHECK(picked[blocks_per_piece - 1].piece_index == 2); + TEST_CHECK(picked[2 * blocks_per_piece - 2].piece_index == 3); + +// ======================================================== + + // make sure the piece picker will pick pieces that + // are already requested from other peers if it has to + print_title("test picking downloading blocks"); + p = setup_picker("1111111", " ", "", ""); + p->mark_as_downloading(piece_block(2,2), &tmp1, piece_picker::fast); + p->mark_as_downloading(piece_block(1,2), &tmp1, piece_picker::slow); + + picked.clear(); + p->pick_pieces(string2vec("*******"), picked, 7 * blocks_per_piece, 0, 0 + , piece_picker::fast, piece_picker::prioritize_partials, empty_vector); + TEST_CHECK(verify_pick(p, picked, true)); + print_pick(picked); + TEST_CHECK(picked.size() == 7 * blocks_per_piece); + + picked.clear(); + p->pick_pieces(string2vec("*******"), picked, 7 * blocks_per_piece, 0, 0 + , piece_picker::fast, piece_picker::prioritize_partials + | piece_picker::rarest_first, empty_vector); + TEST_CHECK(verify_pick(p, picked, true)); + print_pick(picked); + TEST_CHECK(picked.size() == 7 * blocks_per_piece); + + picked.clear(); + p->pick_pieces(string2vec("*******"), picked, 7 * blocks_per_piece, 0, 0 + , piece_picker::fast, piece_picker::rarest_first, empty_vector); + TEST_CHECK(verify_pick(p, picked, true)); + print_pick(picked); + TEST_CHECK(picked.size() == 7 * blocks_per_piece); + +// ======================================================== + + // test clear_peer + print_title("test clear_peer"); + p = setup_picker("1123333", " ", "", ""); + p->mark_as_downloading(piece_block(0, 0), &tmp1, piece_picker::slow); + p->mark_as_downloading(piece_block(0, 1), &tmp2, piece_picker::slow); + p->mark_as_downloading(piece_block(0, 2), &tmp3, piece_picker::slow); + p->mark_as_downloading(piece_block(1, 1), &tmp1, piece_picker::slow); + p->mark_as_downloading(piece_block(2, 1), &tmp2, piece_picker::slow); + p->mark_as_downloading(piece_block(3, 1), &tmp3, piece_picker::slow); + + std::vector dls; + void* expected_dls1[] = {&tmp1, &tmp2, &tmp3, 0}; + void* expected_dls2[] = {0, &tmp1, 0, 0}; + void* expected_dls3[] = {0, &tmp2, 0, 0}; + void* expected_dls4[] = {0, &tmp3, 0, 0}; + void* expected_dls5[] = {&tmp1, 0, &tmp3, 0}; + p->get_downloaders(dls, 0); + TEST_CHECK(std::equal(dls.begin(), dls.end(), expected_dls1)); + p->get_downloaders(dls, 1); + TEST_CHECK(std::equal(dls.begin(), dls.end(), expected_dls2)); + p->get_downloaders(dls, 2); + TEST_CHECK(std::equal(dls.begin(), dls.end(), expected_dls3)); + p->get_downloaders(dls, 3); + TEST_CHECK(std::equal(dls.begin(), dls.end(), expected_dls4)); + + p->clear_peer(&tmp2); + p->get_downloaders(dls, 0); + TEST_CHECK(std::equal(dls.begin(), dls.end(), expected_dls5)); // ======================================================== @@ -505,7 +766,7 @@ int test_main() // ======================================================== - // test have_all and have_none with sequential download + // test have_all and have_none print_title("test have_all and have_none with sequential download"); p = setup_picker("0123333", "* ", "", ""); dc = p->distributed_copies(); @@ -515,17 +776,6 @@ int test_main() dc = p->distributed_copies(); std::cout << "distributed copies: " << dc << std::endl; TEST_CHECK(fabs(dc - (2.f + 5.f / 7.f)) < 0.01f); - p->sequential_download(true); - p->dec_refcount_all(); - dc = p->distributed_copies(); - std::cout << "distributed copies: " << dc << std::endl; - TEST_CHECK(fabs(dc - (1.f + 5.f / 7.f)) < 0.01f); - p->inc_refcount(0); - p->dec_refcount_all(); - dc = p->distributed_copies(); - std::cout << "distributed copies: " << dc << std::endl; - TEST_CHECK(fabs(dc - (0.f + 6.f / 7.f)) < 0.01f); - p->inc_refcount(1); TEST_CHECK(test_pick(p) == 1); // ======================================================== @@ -546,40 +796,15 @@ int test_main() p->dec_refcount(5); p->inc_refcount(5); - p->inc_refcount(0); - p->dec_refcount(4); + bitfield bits(7); + bits.clear_all(); + bits.set_bit(0); + p->inc_refcount(bits); + bits.clear_all(); + bits.set_bit(4); + p->dec_refcount(bits); TEST_CHECK(test_pick(p) == 0); -// ======================================================== -/* - // test have_all and have_none, with a sequenced download threshold - p = setup_picker("1233333", "* ", "", ""); - p->set_sequenced_download_threshold(3); - p->inc_refcount_all(); - dc = p->distributed_copies(); - TEST_CHECK(fabs(dc - (3.f + 5.f / 7.f)) < 0.01f); - p->dec_refcount_all(); - dc = p->distributed_copies(); - TEST_CHECK(fabs(dc - (2.f + 5.f / 7.f)) < 0.01f); - p->dec_refcount(2); - dc = p->distributed_copies(); - TEST_CHECK(fabs(dc - (2.f + 4.f / 7.f)) < 0.01f); - p->mark_as_downloading(piece_block(1,0), &peer_struct, piece_picker::fast); - p->mark_as_downloading(piece_block(1,1), &peer_struct, piece_picker::fast); - p->we_have(1); - dc = p->distributed_copies(); - TEST_CHECK(fabs(dc - (2.f + 5.f / 7.f)) < 0.01f); - picked.clear(); - // make sure it won't pick the piece we just got - p->pick_pieces(string2vec(" * ****"), picked, 100, false, 0, piece_picker::fast, true, false, empty_vector); - print_pick(picked); - TEST_CHECK(verify_pick(p, picked)); - TEST_CHECK(int(picked.size()) >= 4 * blocks_per_piece); - TEST_CHECK(picked[0 * blocks_per_piece].piece_index == 3); - TEST_CHECK(picked[1 * blocks_per_piece].piece_index == 4); - TEST_CHECK(picked[2 * blocks_per_piece].piece_index == 5); - TEST_CHECK(picked[3 * blocks_per_piece].piece_index == 6); -*/ // ======================================================== // test unverified_blocks, marking blocks and get_downloader @@ -592,7 +817,6 @@ int test_main() TEST_CHECK(p->get_downloader(piece_block(4, 3)) == 0); p->mark_as_downloading(piece_block(4, 3), &peer_struct, piece_picker::fast); TEST_CHECK(p->get_downloader(piece_block(4, 3)) == &peer_struct); - piece_picker::downloading_piece st; p->piece_info(4, st); TEST_CHECK(st.requested == 1); TEST_CHECK(st.writing == 0); @@ -625,10 +849,7 @@ int test_main() // test prefer_whole_pieces print_title("test prefer whole pieces"); p = setup_picker("1111111", " ", "", ""); - picked.clear(); - p->pick_pieces(string2vec("*******"), picked, 1, 3, 0, piece_picker::fast, true, false, empty_vector); - print_pick(picked); - TEST_CHECK(verify_pick(p, picked)); + picked = pick_pieces(p, "*******", 1, 3, 0, piece_picker::fast, options, empty_vector); TEST_CHECK(int(picked.size()) >= 3 * blocks_per_piece); piece_block b = picked.front(); for (int i = 1; i < int(picked.size()); ++i) @@ -638,10 +859,7 @@ int test_main() b = picked[i]; } - picked.clear(); - p->pick_pieces(string2vec("*******"), picked, 1, 3, 0, piece_picker::fast, false, false, empty_vector); - print_pick(picked); - TEST_CHECK(verify_pick(p, picked)); + picked = pick_pieces(p, "*******", 1, 3, 0, piece_picker::fast, options, empty_vector); TEST_CHECK(int(picked.size()) >= 3 * blocks_per_piece); b = picked.front(); for (int i = 1; i < int(picked.size()); ++i) @@ -650,6 +868,17 @@ int test_main() == b.piece_index * blocks_per_piece + b.block_index + 1); b = picked[i]; } + + // make sure pieces that don't match the 'whole pieces' requirement + // are picked if there's no other choice + p = setup_picker("1111111", " ", "", ""); + p->mark_as_downloading(piece_block(2,2), &tmp1, piece_picker::fast); + picked = pick_pieces(p, "*******", 7 * blocks_per_piece - 1, 1, 0 + , piece_picker::fast, options, empty_vector); + TEST_CHECK(picked.size() == 7 * blocks_per_piece - 1); + TEST_CHECK(std::find(picked.begin(), picked.end(), piece_block(2,2)) == picked.end()); + +//#error test picking with partial pieces and other peers present so that both backup_pieces and backup_pieces2 are used // ======================================================== @@ -657,28 +886,19 @@ int test_main() print_title("test parole mode"); p = setup_picker("3333133", " ", "", ""); p->mark_as_finished(piece_block(0, 0), 0); - picked.clear(); - p->pick_pieces(string2vec("*******"), picked, 1, 1, 0, piece_picker::fast, true, true, empty_vector); - print_pick(picked); - TEST_CHECK(verify_pick(p, picked)); - TEST_CHECK(int(picked.size()) >= blocks_per_piece - 1); + picked = pick_pieces(p, "*******", 1, 1, 0, piece_picker::fast + , options | piece_picker::on_parole | piece_picker::prioritize_partials, empty_vector); + TEST_CHECK(int(picked.size()) == blocks_per_piece - 1); for (int i = 1; i < int(picked.size()); ++i) - { - TEST_CHECK(picked[i].piece_index == 0); - TEST_CHECK(picked[i].block_index == i + 1); - } + TEST_CHECK(picked[i] == piece_block(0, i + 1)); // make sure that the partial piece is not picked by a // peer that is has not downloaded/requested the other blocks - picked.clear(); - p->pick_pieces(string2vec("*******"), picked, 1, 1, &peer_struct, piece_picker::fast, true, true, empty_vector); - print_pick(picked); - TEST_CHECK(int(picked.size()) >= blocks_per_piece); + picked = pick_pieces(p, "*******", 1, 1, &peer_struct, piece_picker::fast + , options | piece_picker::on_parole | piece_picker::prioritize_partials, empty_vector); + TEST_CHECK(int(picked.size()) == blocks_per_piece); for (int i = 1; i < int(picked.size()); ++i) - { - TEST_CHECK(picked[i].piece_index == 4); - TEST_CHECK(picked[i].block_index == i); - } + TEST_CHECK(picked[i] == piece_block(4, i)); // ======================================================== @@ -688,53 +908,29 @@ int test_main() int v[] = {1, 5}; std::vector suggested_pieces(v, v + 2); - picked.clear(); - p->pick_pieces(string2vec("****************"), picked, 1, 1, 0, piece_picker::fast, true, true, suggested_pieces); - print_pick(picked); - TEST_CHECK(verify_pick(p, picked)); + picked = pick_pieces(p, "****************", 1, 1, 0, piece_picker::fast, options, suggested_pieces); TEST_CHECK(int(picked.size()) >= blocks_per_piece); for (int i = 1; i < int(picked.size()); ++i) - { - TEST_CHECK(picked[i].piece_index == 1); - TEST_CHECK(picked[i].block_index == i); - } + TEST_CHECK(picked[i] == piece_block(1, i)); p->set_piece_priority(0, 0); p->set_piece_priority(1, 0); p->set_piece_priority(2, 0); p->set_piece_priority(3, 0); - picked.clear(); - p->pick_pieces(string2vec("****************"), picked, 1, 1, 0, piece_picker::fast, true, true, suggested_pieces); - print_pick(picked); - TEST_CHECK(verify_pick(p, picked)); + picked = pick_pieces(p, "****************", 1, 1, 0, piece_picker::fast, options, suggested_pieces); TEST_CHECK(int(picked.size()) >= blocks_per_piece); for (int i = 1; i < int(picked.size()); ++i) - { - TEST_CHECK(picked[i].piece_index == 5); - TEST_CHECK(picked[i].block_index == i); - } + TEST_CHECK(picked[i] == piece_block(5, i)); p = setup_picker("1111222233334444", "**** ", "", ""); - picked.clear(); - p->pick_pieces(string2vec("****************"), picked, 1, 1, 0, piece_picker::fast, true, true, suggested_pieces); - print_pick(picked); - TEST_CHECK(verify_pick(p, picked)); + picked = pick_pieces(p, "****************", 1, 1, 0, piece_picker::fast, options, suggested_pieces); TEST_CHECK(int(picked.size()) >= blocks_per_piece); for (int i = 1; i < int(picked.size()); ++i) - { - TEST_CHECK(picked[i].piece_index == 5); - TEST_CHECK(picked[i].block_index == i); - } + TEST_CHECK(picked[i] == piece_block(5, i)); // MISSING TESTS: -// 2. inc_ref() from 0 to 1 while sequenced download threshold is 1 -// 4. filtered_pieces -// 5. clear peer -// 6. pick_pieces with prefer whole pieces -// 7. is_requested -// 8. is_downloaded -// 9. get_downloaders -// 10. abort_download +// 1. abort_download +// 2. write_failed /*