From a15f58934131da8ebe020033b8abecaa8e37c2e2 Mon Sep 17 00:00:00 2001 From: Arvid Norberg Date: Thu, 1 Sep 2005 21:04:21 +0000 Subject: [PATCH] fixed problem in piece_picker with filtered files, also optimzed it somewhat. Added more logging and added one test to the piece_picker test --- examples/client_test.cpp | 4 ++ include/libtorrent/debug.hpp | 5 +- include/libtorrent/piece_picker.hpp | 7 +-- include/libtorrent/session.hpp | 2 +- src/peer_connection.cpp | 5 +- src/piece_picker.cpp | 89 +++++++++++++++++++---------- src/policy.cpp | 4 ++ src/session.cpp | 28 +++++++-- src/storage.cpp | 11 ++-- src/torrent.cpp | 31 +++++++--- test/test_piece_picker.cpp | 6 +- 11 files changed, 133 insertions(+), 59 deletions(-) diff --git a/examples/client_test.cpp b/examples/client_test.cpp index 3e142171e..bf01cc82e 100755 --- a/examples/client_test.cpp +++ b/examples/client_test.cpp @@ -409,6 +409,10 @@ int main(int argc, char* argv[]) handles.back().set_max_connections(60); handles.back().set_max_uploads(-1); // handles.back().set_ratio(1.02f); + +// std::vector ffilter(t.num_files(), true); +// ffilter[0] = false; +// handles.back().filter_files(ffilter); } catch (std::exception& e) diff --git a/include/libtorrent/debug.hpp b/include/libtorrent/debug.hpp index 0f303405c..87d65b4d8 100755 --- a/include/libtorrent/debug.hpp +++ b/include/libtorrent/debug.hpp @@ -103,12 +103,13 @@ namespace libtorrent struct file_logger: libtorrent::logger { public: - file_logger(boost::filesystem::path const& filename) + file_logger(boost::filesystem::path const& filename, bool append = true) { using namespace boost::filesystem; path dir(complete("libtorrent_logs")); if (!exists(dir)) create_directories(dir); - m_file.open(dir / filename); + m_file.open(dir / filename, std::ios_base::out | (append ? std::ios_base::app : std::ios_base::out)); + log("\n\n\n*** starting log ***\n"); } virtual void log(const char* text) { assert(text); m_file << text; } diff --git a/include/libtorrent/piece_picker.hpp b/include/libtorrent/piece_picker.hpp index c75fe0cbd..b178aa6cd 100755 --- a/include/libtorrent/piece_picker.hpp +++ b/include/libtorrent/piece_picker.hpp @@ -260,6 +260,7 @@ namespace libtorrent }; + void add(int index); void move(bool downloading, bool filtered, int vec_index, int elem_index); void remove(bool downloading, bool filtered, int vec_index, int elem_index); std::vector >& pick_piece_info_vector(bool downloading @@ -294,14 +295,12 @@ namespace libtorrent // during piece picking std::vector > m_downloading_piece_info; - // this vector has the same structure as m_piece_info - // but only contains pieces we aren't interested in (filtered) - std::vector > m_filtered_piece_info; - // this maps indices to number of peers that has this piece and // index into the m_piece_info vectors. // piece_pos::we_have_index means that we have the piece, so it // doesn't exist in the piece_info buckets + // pieces with the filtered flag set doesn't have entries in + // the m_piece_info buckets either std::vector m_piece_map; // each piece that's currently being downloaded diff --git a/include/libtorrent/session.hpp b/include/libtorrent/session.hpp index 5101fc729..948bddbd3 100755 --- a/include/libtorrent/session.hpp +++ b/include/libtorrent/session.hpp @@ -271,7 +271,7 @@ namespace libtorrent void check_invariant(const char *place = 0); #endif #if defined(TORRENT_VERBOSE_LOGGING) || defined(TORRENT_LOGGING) - boost::shared_ptr create_log(std::string const& name); + boost::shared_ptr create_log(std::string const& name, bool append = true); boost::shared_ptr m_logger; #endif }; diff --git a/src/peer_connection.cpp b/src/peer_connection.cpp index bcd66b6ed..7083c18d1 100755 --- a/src/peer_connection.cpp +++ b/src/peer_connection.cpp @@ -158,6 +158,7 @@ namespace libtorrent #ifdef TORRENT_VERBOSE_LOGGING m_logger = m_ses.create_log(s->sender().as_string().c_str()); + (*m_logger) << "*** OUTGOING CONNECTION\n"; #endif std::fill(m_peer_id.begin(), m_peer_id.end(), 0); @@ -272,6 +273,7 @@ namespace libtorrent #ifdef TORRENT_VERBOSE_LOGGING m_logger = m_ses.create_log(s->sender().as_string().c_str()); + (*m_logger) << "*** INCOMING CONNECTION\n"; #endif // initialize the extension list to zero, since @@ -795,9 +797,6 @@ namespace libtorrent // if we're a seed too, disconnect if (m_torrent->is_seed()) { -#ifdef TORRENT_VERBOSE_LOGGING - (*m_logger) << " we're also a seed, disconnecting\n"; -#endif throw protocol_error("seed to seed connection redundant, disconnecting"); } } diff --git a/src/piece_picker.cpp b/src/piece_picker.cpp index 153e407b1..e628afa6c 100755 --- a/src/piece_picker.cpp +++ b/src/piece_picker.cpp @@ -48,7 +48,6 @@ namespace libtorrent piece_picker::piece_picker(int blocks_per_piece, int total_num_blocks) : m_piece_info(2) , m_downloading_piece_info(2) - , m_filtered_piece_info(2) , m_piece_map((total_num_blocks + blocks_per_piece-1) / blocks_per_piece) , m_num_filtered(0) , m_num_have_filtered(0) @@ -84,6 +83,10 @@ namespace libtorrent std::vector piece_list; piece_list.reserve(std::count(pieces.begin(), pieces.end(), false)); +#ifndef NDEBUG + integrity_check(); +#endif + for (std::vector::const_iterator i = pieces.begin(); i != pieces.end(); ++i) { @@ -93,10 +96,14 @@ namespace libtorrent { ++m_num_filtered; --m_num_have_filtered; + m_piece_map[index].index = 0; + } + else + { + piece_list.push_back(index); } - piece_list.push_back(index); } - + // random shuffle the list std::random_shuffle(piece_list.begin(), piece_list.end()); @@ -107,20 +114,18 @@ namespace libtorrent int index = *i; assert(index >= 0); assert(index < (int)m_piece_map.size()); - assert(m_piece_map[index].index == piece_pos::we_have_index); - - int peer_count = m_piece_map[index].peer_count; - assert(peer_count == 0); + assert(m_piece_map[index].index == piece_pos::we_have_index); + assert(m_piece_map[index].peer_count == 0); assert(m_piece_info.size() == 2); - piece_pos& p = m_piece_map[index]; - std::vector >& dst_vec = pick_piece_info_vector( - p.downloading, p.filtered); - assert((int)dst_vec.size() > peer_count); - p.index = (int)dst_vec[peer_count].size(); - dst_vec[peer_count].push_back(index); + add(index); + assert(m_piece_map[index].index != piece_pos::we_have_index); } +#ifndef NDEBUG + integrity_check(); +#endif + // if we have fast resume info // use it if (!unfinished.empty()) @@ -225,7 +230,7 @@ namespace libtorrent } } - else + else if (!i->filtered) { if (t != 0) assert(!t->have_piece(index)); @@ -276,24 +281,40 @@ namespace libtorrent std::vector >& piece_picker::pick_piece_info_vector( bool downloading, bool filtered) { - return filtered - ?m_filtered_piece_info - :(downloading?m_downloading_piece_info:m_piece_info); + assert(!filtered); + return downloading?m_downloading_piece_info:m_piece_info; } std::vector > const& piece_picker::pick_piece_info_vector( bool downloading, bool filtered) const { - return filtered - ?m_filtered_piece_info - :(downloading?m_downloading_piece_info:m_piece_info); + assert(!filtered); + return downloading?m_downloading_piece_info:m_piece_info; } + void piece_picker::add(int index) + { + assert(index >= 0); + assert(index < (int)m_piece_map.size()); + piece_pos& p = m_piece_map[index]; + assert(!p.filtered); + + std::vector >& dst_vec = pick_piece_info_vector( + p.downloading, p.filtered); + + if (dst_vec.size() <= p.peer_count) + dst_vec.resize(p.peer_count + 1); + + assert(dst_vec.size() > p.peer_count); + p.index = (int)dst_vec[p.peer_count].size(); + dst_vec[p.peer_count].push_back(index); + } // will update the piece with the given properties (downloading, filtered, peer_count, elem_index) // to place it at the correct position in the vectors. void piece_picker::move(bool downloading, bool filtered, int peer_count, int elem_index) { + assert(!filtered); assert(peer_count >= 0); assert(elem_index >= 0); std::vector >& src_vec(pick_piece_info_vector(downloading, filtered)); @@ -345,6 +366,7 @@ namespace libtorrent void piece_picker::remove(bool downloading, bool filtered, int peer_count, int elem_index) { + assert(!filtered); assert(peer_count >= 0); assert(elem_index >= 0); @@ -354,7 +376,6 @@ namespace libtorrent assert((int)src_vec[peer_count].size() > elem_index); int index = src_vec[peer_count][elem_index]; - m_piece_map[index].index = piece_pos::we_have_index; if (downloading) { @@ -393,6 +414,7 @@ namespace libtorrent m_piece_map[index].downloading = 0; piece_pos& p = m_piece_map[index]; + if (p.filtered) return; move(true, p.filtered, p.peer_count, p.index); #ifndef NDEBUG @@ -412,11 +434,12 @@ namespace libtorrent m_piece_map[i].peer_count++; assert(m_piece_map[i].peer_count != 0); - // if we have the piece, we don't have to move - // any entries in the piece_info vector - if (index == piece_pos::we_have_index) return; - piece_pos& p = m_piece_map[i]; + + // if we have the piece or if it's filtered + // we don't have to move any entries in the piece_info vector + if (index == piece_pos::we_have_index || p.filtered) return; + move(p.downloading, p.filtered, peer_count, index); #ifndef NDEBUG @@ -441,8 +464,10 @@ namespace libtorrent if (m_piece_map[i].peer_count > 0) m_piece_map[i].peer_count--; - if (index == piece_pos::we_have_index) return; piece_pos& p = m_piece_map[i]; + + if (index == piece_pos::we_have_index || p.filtered) return; + move(p.downloading, p.filtered, peer_count, index); } @@ -466,8 +491,11 @@ namespace libtorrent { --m_num_filtered; ++m_num_have_filtered; + return; } + if (info_index == piece_pos::we_have_index) return; remove(p.downloading, p.filtered, peer_count, info_index); + p.index = piece_pos::we_have_index; #ifndef NDEBUG integrity_check(); #endif @@ -480,7 +508,7 @@ namespace libtorrent assert(index < (int)m_piece_map.size()); #ifndef NDEBUG - integrity_check(); +// integrity_check(); #endif piece_pos& p = m_piece_map[index]; @@ -489,7 +517,8 @@ namespace libtorrent if (p.index != piece_pos::we_have_index) { ++m_num_filtered; - move(p.downloading, false, p.peer_count, p.index); + remove(p.downloading, false, p.peer_count, p.index); + assert(p.filtered == 1); } else { @@ -518,7 +547,7 @@ namespace libtorrent { --m_num_filtered; assert(m_num_filtered >= 0); - move(p.downloading, true, p.peer_count, p.index); + add(index); } else { @@ -854,7 +883,7 @@ namespace libtorrent assert(block.block_index < blocks_in_piece(block.piece_index)); piece_pos& p = m_piece_map[block.piece_index]; - if (p.index == piece_pos::we_have_index) return; + if (p.index == piece_pos::we_have_index || p.filtered) return; if (p.downloading == 0) { diff --git a/src/policy.cpp b/src/policy.cpp index 3c06892f5..e15544aa2 100755 --- a/src/policy.cpp +++ b/src/policy.cpp @@ -1147,6 +1147,10 @@ namespace libtorrent peer *p = find_disconnect_candidate(); if(!p) return false; +#if defined(TORRENT_VERBOSE_LOGGING) + (*p->connection->m_logger) << "*** CLOSING CONNECTION 'too many connections'\n"; +#endif + p->connection->disconnect(); return true; } diff --git a/src/session.cpp b/src/session.cpp index 42ec7e9a7..8b4b96dbd 100755 --- a/src/session.cpp +++ b/src/session.cpp @@ -234,7 +234,7 @@ namespace libtorrent { namespace detail , m_incoming_connection(false) { #if defined(TORRENT_VERBOSE_LOGGING) || defined(TORRENT_LOGGING) - m_logger = create_log("main_session"); + m_logger = create_log("main_session", false); #endif std::fill(m_extension_enabled, m_extension_enabled + peer_connection::num_supported_extensions, true); @@ -257,8 +257,7 @@ namespace libtorrent { namespace detail // the random number for (unsigned char* i = m_peer_id.begin() + print.length(); - i != m_peer_id.end(); - ++i) + i != m_peer_id.end(); ++i) { *i = printable[rand() % (sizeof(printable)-1)]; } @@ -480,6 +479,10 @@ namespace libtorrent { namespace detail , e.what())); } +#if defined(TORRENT_VERBOSE_LOGGING) + (*p->second->m_logger) << "*** CLOSING CONNECTION '" << e.what() << "'\n"; +#endif + p->second->set_failed(); m_selector.remove(*i); m_connections.erase(p); @@ -576,6 +579,11 @@ namespace libtorrent { namespace detail , p->second->id() , e.what())); } + +#if defined(TORRENT_VERBOSE_LOGGING) + (*p->second->m_logger) << "*** CLOSING CONNECTION '" << e.what() << "'\n"; +#endif + // the connection wants to disconnect for some reason, remove it // from the connection-list p->second->set_failed(); @@ -615,6 +623,9 @@ namespace libtorrent { namespace detail // the connection may have been disconnected in the receive or send phase if (p != m_connections.end()) { +#if defined(TORRENT_VERBOSE_LOGGING) + (*p->second->m_logger) << "*** CONNECTION EXCEPTION\n"; +#endif p->second->set_failed(); m_connections.erase(p); } @@ -667,6 +678,10 @@ namespace libtorrent { namespace detail , j->second->id() , "connection timed out")); } +#if defined(TORRENT_VERBOSE_LOGGING) + (*j->second->m_logger) << "*** CONNECTION TIMED OUT\n"; +#endif + j->second->set_failed(); m_selector.remove(j->first); m_connections.erase(j); @@ -810,10 +825,10 @@ namespace libtorrent { namespace detail } #if defined(TORRENT_VERBOSE_LOGGING) || defined(TORRENT_LOGGING) - boost::shared_ptr session_impl::create_log(std::string const& name) + boost::shared_ptr session_impl::create_log(std::string const& name, bool append) { // current options are file_logger, cout_logger and null_logger - return boost::shared_ptr(new file_logger(name + ".log")); + return boost::shared_ptr(new file_logger(name + ".log", append)); } #endif @@ -908,6 +923,9 @@ namespace libtorrent if (m_impl.m_ip_filter.access(i->first->sender()) & ip_filter::blocked) { +#if defined(TORRENT_VERBOSE_LOGGING) + (*i->second->m_logger) << "*** CONNECTION FILTERED'\n"; +#endif m_impl.m_connections.erase(i++); } else ++i; diff --git a/src/storage.cpp b/src/storage.cpp index 5d4634eab..5b155e635 100755 --- a/src/storage.cpp +++ b/src/storage.cpp @@ -30,11 +30,7 @@ POSSIBILITY OF SUCH DAMAGE. */ -#include #include -#include -#include -#include #include #include #include @@ -70,6 +66,13 @@ POSSIBILITY OF SUCH DAMAGE. #include "libtorrent/file.hpp" #include "libtorrent/invariant_check.hpp" +#ifndef NDEBUG +#include +#include +#include +#include +#endif + #if defined(_WIN32) && defined(UNICODE) #include diff --git a/src/torrent.cpp b/src/torrent.cpp index bbfb4bd63..31261f442 100755 --- a/src/torrent.cpp +++ b/src/torrent.cpp @@ -579,6 +579,10 @@ namespace libtorrent , "banning peer because of too many corrupt pieces")); } m_policy->ban_peer(*p->second); + +#if defined(TORRENT_VERBOSE_LOGGING) + (*p->second->m_logger) << "*** BANNING PEER 'too many corrupt pieces'\n"; +#endif p->second->disconnect(); } } @@ -661,21 +665,23 @@ namespace libtorrent // TODO: update peer's interesting-bit - std::vector > state; + std::vector state; state.reserve(100); int index = 0; for (std::vector::const_iterator i = bitmask.begin() , end(bitmask.end()); i != end; ++i, ++index) { if (m_picker->is_filtered(index) == *i) continue; - state.push_back(std::make_pair(index, *i)); + if (*i) + m_picker->mark_as_filtered(index); + else + state.push_back(index); } std::random_shuffle(state.begin(), state.end()); - for (std::vector >::iterator i = state.begin(); + for (std::vector::iterator i = state.begin(); i != state.end(); ++i) { - if (i->second) m_picker->mark_as_filtered(i->first); - else m_picker->mark_as_unfiltered(i->first); + m_picker->mark_as_unfiltered(*i); } } @@ -892,10 +898,16 @@ namespace libtorrent void torrent::disconnect_all() { for (peer_iterator i = m_connections.begin(); - i != m_connections.end(); - ++i) + i != m_connections.end(); ++i) { assert(i->second->associated_torrent() == this); + +#if defined(TORRENT_VERBOSE_LOGGING) + if (m_abort) + (*i->second->m_logger) << "*** CLOSING CONNECTION 'aborting'\n"; + else + (*i->second->m_logger) << "*** CLOSING CONNECTION 'pausing'\n"; +#endif i->second->disconnect(); } } @@ -916,7 +928,12 @@ namespace libtorrent { assert(i->second->associated_torrent() == this); if (i->second->is_seed()) + { +#if defined(TORRENT_VERBOSE_LOGGING) + (*i->second->m_logger) << "*** SEED, CLOSING CONNECTION\n"; +#endif i->second->disconnect(); + } } m_storage->release_files(); diff --git a/test/test_piece_picker.cpp b/test/test_piece_picker.cpp index 11d251b8a..31c13fe5a 100644 --- a/test/test_piece_picker.cpp +++ b/test/test_piece_picker.cpp @@ -243,9 +243,9 @@ int test_main() TEST_CHECK(std::equal(picked.begin() , picked.end(), expected6)); - - // TODO: make sure there are no duplicates - // in the picked blocks! + // make sure the piece picker allows filtered pieces + // to become available + p.mark_as_finished(piece_block(4, 2), address()); } return 0;