added peer/piece categories to the piece picker. fixes #18

This commit is contained in:
Arvid Norberg 2007-04-27 00:27:37 +00:00
parent 99dc46bd29
commit ee1681e2cb
10 changed files with 203 additions and 48 deletions

View File

@ -1699,6 +1699,8 @@ requested. The entry in the vector (``partial_piece_info``) looks like this::
std::bitset<max_blocks_per_piece> finished_blocks;
address peer[max_blocks_per_piece];
int num_downloads[max_blocks_per_piece];
enum state_t { none, slow. medium, fast };
state_t piece_state;
};
``piece_index`` is the index of the piece in question. ``blocks_in_piece`` is the
@ -1718,6 +1720,13 @@ or not. And the ``num_downloads`` array says how many times that block has been
When a piece fails a hash verification, single blocks may be re-downloaded to
see if the hash test may pass then.
``piece_state`` is set to either ``fast``, ``medium``, ``slow`` or ``none``. It tells which
download rate category the peers downloading this piece falls into. ``none`` means that no
peer is currently downloading any part of the piece. Peers prefer picking pieces from
the same category as themselves. The reason for this is to keep the number of partially
downloaded pieces down. Pieces set to ``none`` can be converted into any of ``fast``,
``medium`` or ``slow`` as soon as a peer want to download from it.
get_peer_info()
---------------

View File

@ -995,7 +995,8 @@ int main(int ac, char* av[])
else out << "-";
#endif
}
out << "]\n";
char* piece_state[4] = {"", "slow", "medium", "fast"};
out << "] " << piece_state[i->piece_state] << "\n";
}
out << "___________________________________\n";

View File

@ -136,6 +136,9 @@ namespace libtorrent
policy::peer* peer_info_struct() const
{ return m_peer_info; }
enum peer_speed_t { slow, medium, fast };
peer_speed_t peer_speed();
#ifndef TORRENT_DISABLE_EXTENSIONS
void add_extension(boost::shared_ptr<peer_plugin>);
#endif
@ -650,6 +653,10 @@ namespace libtorrent
// and hasn't been added to a torrent yet.
policy::peer* m_peer_info;
// this is a measurement of how fast the peer
// it allows some variance without changing
// back and forth between states
peer_speed_t m_speed;
#ifndef NDEBUG
public:
bool m_in_constructor;

View File

@ -100,8 +100,18 @@ namespace libtorrent
int num_downloads;
};
// the peers that are downloading this piece
// are considered fast peers or slow peers.
// none is set if the blocks were downloaded
// in a previous session
enum piece_state_t
{ none, slow, medium, fast };
struct downloading_piece
{
piece_state_t state;
// the index of the piece
int index;
// each bit represents a block in the piece
// set to one if the block has been requested
@ -173,7 +183,7 @@ namespace libtorrent
void pick_pieces(const std::vector<bool>& pieces
, std::vector<piece_block>& interesting_blocks
, int num_pieces, bool prefer_whole_pieces
, tcp::endpoint peer) const;
, tcp::endpoint peer, piece_state_t speed) const;
// returns true if any client is currently downloading this
// piece-block, or if it's queued for downloading by some client
@ -182,7 +192,8 @@ namespace libtorrent
bool is_finished(piece_block block) const;
// marks this piece-block as queued for downloading
void mark_as_downloading(piece_block block, tcp::endpoint const& peer);
void mark_as_downloading(piece_block block, tcp::endpoint const& peer
, piece_state_t s);
void mark_as_finished(piece_block block, tcp::endpoint const& peer);
// if a piece had a hash-failure, it must be restored and
@ -320,15 +331,13 @@ namespace libtorrent
void add(int index);
void move(int vec_index, int elem_index);
// void remove(int vec_index, int elem_index);
int add_interesting_blocks(const std::vector<int>& piece_list
, const std::vector<bool>& pieces
, std::vector<piece_block>& interesting_blocks
, std::vector<piece_block>& backup_blocks
, int num_blocks, bool prefer_whole_pieces
, tcp::endpoint peer) const;
, tcp::endpoint peer, piece_state_t speed) const;
// this vector contains all pieces we don't have.
// in the first entry (index 0) is a vector of all pieces

View File

@ -211,6 +211,8 @@ namespace libtorrent
std::bitset<max_blocks_per_piece> finished_blocks;
tcp::endpoint peer[max_blocks_per_piece];
int num_downloads[max_blocks_per_piece];
enum state_t { none, slow, medium, fast };
state_t piece_state;
};
struct TORRENT_EXPORT torrent_handle

View File

@ -124,6 +124,7 @@ namespace libtorrent
, m_upload_limit(resource_request::inf)
, m_download_limit(resource_request::inf)
, m_peer_info(peerinfo)
, m_speed(slow)
#ifndef NDEBUG
, m_in_constructor(true)
#endif
@ -189,6 +190,7 @@ namespace libtorrent
, m_upload_limit(resource_request::inf)
, m_download_limit(resource_request::inf)
, m_peer_info(peerinfo)
, m_speed(slow)
#ifndef NDEBUG
, m_in_constructor(true)
#endif
@ -1328,7 +1330,14 @@ namespace libtorrent
assert(block.block_index < t->torrent_file().piece_size(block.piece_index));
assert(!t->picker().is_downloading(block));
t->picker().mark_as_downloading(block, m_remote);
piece_picker::piece_state_t state;
peer_speed_t speed = peer_speed();
if (speed == fast) state = piece_picker::fast;
else if (speed == medium) state = piece_picker::medium;
else if (speed == slow) state = piece_picker::slow;
t->picker().mark_as_downloading(block, m_remote, state);
m_request_queue.push_back(block);
}
@ -2411,6 +2420,25 @@ namespace libtorrent
return false;
}
peer_connection::peer_speed_t peer_connection::peer_speed()
{
shared_ptr<torrent> t = m_torrent.lock();
assert(t);
int download_rate = statistics().download_payload_rate();
int torrent_download_rate = t->statistics().download_payload_rate();
if (download_rate > 512 && download_rate > torrent_download_rate / 16)
m_speed = fast;
else if (download_rate > 4096 && download_rate > torrent_download_rate / 64)
m_speed = medium;
else if (download_rate < torrent_download_rate / 15 && m_speed == fast)
m_speed = medium;
else if (download_rate < torrent_download_rate / 63 && m_speed == medium)
m_speed = slow;
return m_speed;
}
void peer_connection::keep_alive()
{

View File

@ -205,6 +205,24 @@ namespace libtorrent
for (int i = m_sequenced_download_threshold * 2 + 1; i < int(m_piece_info.size()); ++i)
assert(m_piece_info[i].empty());
for (std::vector<downloading_piece>::const_iterator i = m_downloads.begin()
, end(m_downloads.end()); i != end; ++i)
{
bool blocks_requested = false;
int num_blocks = blocks_in_piece(i->index);
for (int k = 0; k < num_blocks; ++k)
{
if (i->finished_blocks[k]) continue;
if (i->requested_blocks[k])
{
blocks_requested = true;
break;
}
}
assert(blocks_requested == (i->state != none));
}
int num_filtered = 0;
int num_have_filtered = 0;
for (std::vector<piece_pos>::const_iterator i = m_piece_map.begin();
@ -971,28 +989,40 @@ namespace libtorrent
}
// ============ end deprecation ==============
// pieces describes which pieces the peer we're requesting from
// has.
// interesting_blocks is an out parameter, and will be filled
// with (up to) num_blocks of interesting blocks that the peer has.
// prefer_whole_pieces can be set if this peer should download
// whole pieces rather than trying to download blocks from the
// same piece as other peers.
// the endpoint is the address of the peer we're picking pieces
// from. This is used when downloading whole pieces, to only
// pick from the same piece the same peer is downloading from.
// state is supposed to be set to fast if the peer is downloading
// relatively fast, by some notion. Slow peers will prefer not
// to pick blocks from the same pieces as fast peers, and vice
// versa. Downloading pieces are marked as being fast, medium
// or slow once they're started.
void piece_picker::pick_pieces(const std::vector<bool>& pieces
, std::vector<piece_block>& interesting_blocks
, int num_blocks, bool prefer_whole_pieces
, tcp::endpoint peer) const
, tcp::endpoint peer, piece_state_t speed) const
{
TORRENT_PIECE_PICKER_INVARIANT_CHECK;
assert(num_blocks > 0);
assert(pieces.size() == m_piece_map.size());
assert(m_files_checked_called);
// free refers to pieces that are free to download, no one else
// is downloading them.
// partial is pieces that are partially being downloaded, and
// parts of them may be free for download as well, the
// partially downloaded pieces will be prioritized
assert(m_piece_info.begin() != m_piece_info.end());
// +1 is to ignore pieces that no peer has. The bucket with index 0 contains
// pieces that 0 other peers has.
std::vector<std::vector<int> >::const_iterator free
= m_piece_info.begin() + 1;
// this will be filled with blocks that we should not request
// unless we can't find num_blocks among the other ones.
// blocks that belong to pieces with a mismatching speed
// category for instance, or if we prefer whole pieces,
// blocks belonging to a piece that others have
// downloaded to
std::vector<piece_block> backup_blocks;
// this loop will loop from pieces with priority 1 and up
@ -1004,29 +1034,29 @@ namespace libtorrent
// fast peers) the partial pieces will not be prioritized, but actually
// ignored as long as possible.
while (free != m_piece_info.end())
// +1 is to ignore pieces that no peer has. The bucket with index 0 contains
// pieces that 0 other peers have. bucket will point to a bucket with
// pieces with the same priority. It will be iterated in priority
// order (high priority/rare pices first). The content of each
// bucket is randomized
for (std::vector<std::vector<int> >::const_iterator bucket
= m_piece_info.begin() + 1; bucket != m_piece_info.end();
++bucket)
{
num_blocks = add_interesting_blocks(*free, pieces
if (bucket->empty()) continue;
num_blocks = add_interesting_blocks(*bucket, pieces
, interesting_blocks, backup_blocks, num_blocks
, prefer_whole_pieces, peer);
, prefer_whole_pieces, peer, speed);
assert(num_blocks >= 0);
if (num_blocks == 0) return;
++free;
}
// TODO: what's up with this?
if (!prefer_whole_pieces) return;
assert(num_blocks > 0);
#ifdef TORRENT_VERBOSE_LOGGING
// std::ofstream f("piece_picker.log", std::ios_base::app);
// f << "backup_blocks: " << backup_blocks.size() << "\n"
// << "used: " << std::min(num_blocks, (int)backup_blocks.size()) << "\n----\n";
#endif
interesting_blocks.insert(interesting_blocks.end()
, backup_blocks.begin(), backup_blocks.begin()
+ (std::min)(num_blocks, (int)backup_blocks.size()));
if (!backup_blocks.empty())
interesting_blocks.insert(interesting_blocks.end()
, backup_blocks.begin(), backup_blocks.begin()
+ (std::min)(num_blocks, (int)backup_blocks.size()));
}
namespace
@ -1053,7 +1083,7 @@ namespace libtorrent
, std::vector<piece_block>& interesting_blocks
, std::vector<piece_block>& backup_blocks
, int num_blocks, bool prefer_whole_pieces
, tcp::endpoint peer) const
, tcp::endpoint peer, piece_state_t speed) const
{
for (std::vector<int>::const_iterator i = piece_list.begin();
i != piece_list.end(); ++i)
@ -1083,7 +1113,7 @@ namespace libtorrent
if (prefer_whole_pieces
&& !exclusively_requested_from(*p, num_blocks_in_piece, peer))
{
if ((int)backup_blocks.size() >= num_blocks) continue;
if (int(backup_blocks.size()) >= num_blocks) continue;
for (int j = 0; j < num_blocks_in_piece; ++j)
{
if (p->finished_blocks[j] == 1) continue;
@ -1096,9 +1126,19 @@ namespace libtorrent
for (int j = 0; j < num_blocks_in_piece; ++j)
{
// ignore completed blocks
if (p->finished_blocks[j] == 1) continue;
// ignore blocks requested from this peer already
if (p->requested_blocks[j] == 1
&& p->info[j].peer == peer) continue;
// if the piece is fast and the peer is slow, or vice versa,
// add the block as a backup
if (p->state != none && p->state != speed)
{
if (int(backup_blocks.size()) >= num_blocks) continue;
backup_blocks.push_back(piece_block(*i, j));
continue;
}
// this block is interesting (we don't have it
// yet). But it may already have been requested
// from another peer. We have to add it anyway
@ -1113,6 +1153,8 @@ namespace libtorrent
{
// we have found a block that's free to download
num_blocks--;
// if we prefer whole pieces, continue picking from this
// piece even though we have num_blocks
if (prefer_whole_pieces) continue;
assert(num_blocks >= 0);
if (num_blocks == 0) return num_blocks;
@ -1193,7 +1235,8 @@ namespace libtorrent
}
void piece_picker::mark_as_downloading(piece_block block, const tcp::endpoint& peer)
void piece_picker::mark_as_downloading(piece_block block
, const tcp::endpoint& peer, piece_state_t state)
{
TORRENT_PIECE_PICKER_INVARIANT_CHECK;
@ -1210,6 +1253,7 @@ namespace libtorrent
move(prio, p.index);
downloading_piece dp;
dp.state = state;
dp.index = block.piece_index;
dp.requested_blocks[block.block_index] = 1;
dp.info[block.block_index].peer = peer;
@ -1223,6 +1267,7 @@ namespace libtorrent
assert(i->requested_blocks[block.block_index] == 0);
i->info[block.block_index].peer = peer;
i->requested_blocks[block.block_index] = 1;
if (i->state == none) i->state = state;
}
}
@ -1245,6 +1290,7 @@ namespace libtorrent
else assert(p.priority(m_sequenced_download_threshold) == 0);
downloading_piece dp;
dp.state = none;
dp.index = block.piece_index;
dp.requested_blocks[block.block_index] = 1;
dp.finished_blocks[block.block_index] = 1;
@ -1259,6 +1305,27 @@ namespace libtorrent
i->info[block.block_index].peer = peer;
i->requested_blocks[block.block_index] = 1;
i->finished_blocks[block.block_index] = 1;
// TODO: maintain requested and finished counters so that
// we don't have to count every time
bool blocks_requested = false;
int num_blocks = blocks_in_piece(i->index);
for (int k = 0; k < num_blocks; ++k)
{
if (i->finished_blocks[k]) continue;
if (i->requested_blocks[k])
{
blocks_requested = true;
break;
}
}
if (!blocks_requested)
{
// there are no blocks requested in this piece.
// remove the fast/slow state from it
i->state = none;
}
}
}
/*
@ -1373,6 +1440,27 @@ namespace libtorrent
assert(std::find_if(m_downloads.begin(), m_downloads.end()
, has_index(block.piece_index)) == m_downloads.end());
}
// TODO: maintain requested and finished counters so that
// we don't have to count every time
bool blocks_requested = false;
int num_blocks = blocks_in_piece(i->index);
for (int k = 0; k < num_blocks; ++k)
{
if (i->finished_blocks[k]) continue;
if (i->requested_blocks[k])
{
blocks_requested = true;
break;
}
}
if (!blocks_requested)
{
// there are no blocks requested in this piece.
// remove the fast/slow state from it
i->state = none;
}
}
int piece_picker::unverified_blocks() const

View File

@ -200,6 +200,12 @@ namespace libtorrent
// than we requested.
assert(c.remote() == c.get_socket()->remote_endpoint());
piece_picker::piece_state_t state;
peer_connection::peer_speed_t speed = c.peer_speed();
if (speed == peer_connection::fast) state = piece_picker::fast;
else if (speed == peer_connection::medium) state = piece_picker::medium;
else if (speed == peer_connection::slow) state = piece_picker::slow;
// picks the interesting pieces from this peer
// the integer is the number of pieces that
// should be guaranteed to be available for download
@ -209,7 +215,7 @@ namespace libtorrent
// for this peer. If we're downloading one piece in 20 seconds
// then use this mode.
p.pick_pieces(c.get_bitfield(), interesting_pieces
, num_requests, prefer_whole_pieces, c.remote());
, num_requests, prefer_whole_pieces, c.remote(), state);
// this vector is filled with the interesting pieces
// that some other peer is currently downloading

View File

@ -764,6 +764,7 @@ namespace libtorrent
= q.begin(); i != q.end(); ++i)
{
partial_piece_info pi;
pi.piece_state = (partial_piece_info::state_t)i->state;
pi.finished_blocks = i->finished_blocks;
pi.requested_blocks = i->requested_blocks;
for (int j = 0; j < partial_piece_info::max_blocks_per_piece; ++j)

View File

@ -88,21 +88,21 @@ int test_main()
std::vector<piece_block> picked;
picked.clear();
p.pick_pieces(peer1, picked, 1, false, tcp::endpoint());
p.pick_pieces(peer1, picked, 1, false, tcp::endpoint(), piece_picker::fast);
TEST_CHECK(picked.size() == 1);
TEST_CHECK(picked.front().piece_index == 2);
// now pick a piece from peer2. The block is supposed to be
// from piece 3, since it is the rarest piece that peer has.
picked.clear();
p.pick_pieces(peer2, picked, 1, false, tcp::endpoint());
p.pick_pieces(peer2, picked, 1, false, tcp::endpoint(), piece_picker::fast);
TEST_CHECK(picked.size() == 1);
TEST_CHECK(picked.front().piece_index == 3);
// same thing for peer3.
picked.clear();
p.pick_pieces(peer3, picked, 1, false, tcp::endpoint());
p.pick_pieces(peer3, picked, 1, false, tcp::endpoint(), piece_picker::fast);
TEST_CHECK(picked.size() == 1);
TEST_CHECK(picked.front().piece_index == 5);
@ -112,7 +112,7 @@ int test_main()
p.inc_refcount(1);
picked.clear();
p.pick_pieces(peer3, picked, 1, false, tcp::endpoint());
p.pick_pieces(peer3, picked, 1, false, tcp::endpoint(), piece_picker::fast);
TEST_CHECK(picked.size() == 1);
TEST_CHECK(picked.front().piece_index == 1);
// and the block picked should not be 0 or 2
@ -136,9 +136,12 @@ int test_main()
// we have block 0 and 2 already, so we can't mark
// them as begin downloaded.
p.mark_as_downloading(piece_block(1, 1), tcp::endpoint(address::from_string("1.1.1.1"), 0));
p.mark_as_downloading(piece_block(1, 3), tcp::endpoint(address::from_string("1.1.1.1"), 0));
p.mark_as_downloading(piece_block(2, 0), tcp::endpoint(address::from_string("1.1.1.1"), 0));
p.mark_as_downloading(piece_block(1, 1), tcp::endpoint(
address::from_string("1.1.1.1"), 0), piece_picker::fast);
p.mark_as_downloading(piece_block(1, 3), tcp::endpoint(
address::from_string("1.1.1.1"), 0), piece_picker::fast);
p.mark_as_downloading(piece_block(2, 0), tcp::endpoint(
address::from_string("1.1.1.1"), 0), piece_picker::fast);
std::vector<piece_picker::downloading_piece> const& downloads = p.get_download_queue();
TEST_CHECK(downloads.size() == 2);
@ -166,7 +169,7 @@ int test_main()
TEST_CHECK(!p.is_downloading(piece_block(2, 1)));
picked.clear();
p.pick_pieces(peer1, picked, 1, false, tcp::endpoint());
p.pick_pieces(peer1, picked, 1, false, tcp::endpoint(), piece_picker::fast);
TEST_CHECK(picked.size() == 2);
piece_block expected3[] = { piece_block(2, 0), piece_block(2, 1) };
@ -179,7 +182,7 @@ int test_main()
// partially selected)
picked.clear();
p.pick_pieces(peer1, picked, 1, true, tcp::endpoint());
p.pick_pieces(peer1, picked, 1, true, tcp::endpoint(), piece_picker::fast);
// it will pick 4 blocks, since we said we
// wanted whole pieces.
@ -197,7 +200,7 @@ int test_main()
// to make sure it can still fall back on partial pieces
picked.clear();
p.pick_pieces(peer1, picked, 100, true, tcp::endpoint());
p.pick_pieces(peer1, picked, 100, true, tcp::endpoint(), piece_picker::fast);
TEST_CHECK(picked.size() == 12);
@ -218,7 +221,8 @@ int test_main()
// to make sure it can still fall back on partial pieces
picked.clear();
p.pick_pieces(peer1, picked, 100, true, tcp::endpoint(address::from_string("1.1.1.1"), 0));
p.pick_pieces(peer1, picked, 100, true
, tcp::endpoint(address::from_string("1.1.1.1"), 0), piece_picker::fast);
TEST_CHECK(picked.size() == 11);