diff --git a/ChangeLog b/ChangeLog index 74e44f5c2..d6e2a090a 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,4 @@ + * support banning web seeds sending corrupt data * don't let hung outgoing connection attempts block incoming connections * improve SSL torrent support by using SNI and a single SSL listen socket * improved peer exchange performance by sharing incoming connections which advertize listen port diff --git a/include/libtorrent/policy.hpp b/include/libtorrent/policy.hpp index 2c4d6ee52..f2243f65a 100644 --- a/include/libtorrent/policy.hpp +++ b/include/libtorrent/policy.hpp @@ -162,7 +162,7 @@ namespace libtorrent // 40 1 1 fast_reconnects, trust_points // 41 1 1 source, pe_support, is_v6_addr // 42 1 1 on_parole, banned, added_to_dht, supports_utp, -// supports_holepunch +// supports_holepunch, web_seed // 43 1 1 // 44 struct TORRENT_EXTRA_EXPORT peer @@ -311,6 +311,12 @@ namespace libtorrent // we have been connected via uTP at least once bool confirmed_supports_utp:1; bool supports_holepunch:1; + // this is set to one for web seeds. Web seeds + // are not stored in the policy m_peers list, + // and are excempt from connect candidate bookkeeping + // so, any peer with the web_seed bit set, is + // never considered a connect candidate + bool web_seed:1; #if defined TORRENT_DEBUG || TORRENT_RELEASE_ASSERTS bool in_use:1; #endif diff --git a/include/libtorrent/torrent.hpp b/include/libtorrent/torrent.hpp index 74e736309..9d4e63489 100644 --- a/include/libtorrent/torrent.hpp +++ b/include/libtorrent/torrent.hpp @@ -370,42 +370,12 @@ namespace libtorrent m_web_seeds.push_back(web_seed_entry(url, type, auth, extra_headers)); } - void remove_web_seed(std::string const& url, web_seed_entry::type_t type) - { - std::list::iterator i = std::find_if(m_web_seeds.begin(), m_web_seeds.end() - , (boost::bind(&web_seed_entry::url, _1) - == url && boost::bind(&web_seed_entry::type, _1) == type)); - if (i != m_web_seeds.end()) remove_web_seed(i); - } - - void disconnect_web_seed(peer_connection* p) - { - std::list::iterator i = std::find_if(m_web_seeds.begin(), m_web_seeds.end() - , (boost::bind(&web_seed_entry::connection, _1) == p)); - // this happens if the web server responded with a redirect - // or with something incorrect, so that we removed the web seed - // immediately, before we disconnected - if (i == m_web_seeds.end()) return; - - TORRENT_ASSERT(i->resolving == false); - -#if defined TORRENT_VERBOSE_LOGGING || defined TORRENT_ERROR_LOGGING - (*m_ses.m_logger) << time_now_string() << " disconnect_web_seed: " << i->url << "\n"; -#endif - TORRENT_ASSERT(i->connection); - i->connection = 0; - } + void remove_web_seed(std::string const& url, web_seed_entry::type_t type); + void disconnect_web_seed(peer_connection* p); void retry_web_seed(peer_connection* p, int retry = 0); - void remove_web_seed(peer_connection* p) - { - std::list::iterator i = std::find_if(m_web_seeds.begin(), m_web_seeds.end() - , (boost::bind(&web_seed_entry::connection, _1) == p)); - TORRENT_ASSERT(i != m_web_seeds.end()); - if (i == m_web_seeds.end()) return; - m_web_seeds.erase(i); - } + void remove_web_seed(peer_connection* p); std::list web_seeds() const { return m_web_seeds; } diff --git a/include/libtorrent/torrent_info.hpp b/include/libtorrent/torrent_info.hpp index 2fd009134..1876f3654 100644 --- a/include/libtorrent/torrent_info.hpp +++ b/include/libtorrent/torrent_info.hpp @@ -58,6 +58,7 @@ POSSIBILITY OF SUCH DAMAGE. #include "libtorrent/file_storage.hpp" #include "libtorrent/copy_ptr.hpp" #include "libtorrent/socket.hpp" +#include "libtorrent/policy.hpp" // for policy::peer namespace libtorrent { @@ -171,12 +172,7 @@ namespace libtorrent web_seed_entry(std::string const& url_, type_t type_ , std::string const& auth_ = std::string() - , headers_t const& extra_headers_ = headers_t()) - : url(url_), type(type_) - , auth(auth_), extra_headers(extra_headers_) - , retry(time_now()), resolving(false), removed(false) - , connection(0) - {} + , headers_t const& extra_headers_ = headers_t()); bool operator==(web_seed_entry const& e) const { return url == e.url && type == e.type; } @@ -208,7 +204,11 @@ namespace libtorrent tcp::endpoint endpoint; - peer_connection* connection; + // this is the peer_info field used for the + // connection, just to count hash failures + // it's also used to hold the peer_connection + // pointer, when the web seed is connected + policy::peer peer_info; }; #ifndef BOOST_NO_EXCEPTIONS diff --git a/src/peer_connection.cpp b/src/peer_connection.cpp index bb09e1506..2151fc157 100644 --- a/src/peer_connection.cpp +++ b/src/peer_connection.cpp @@ -3503,6 +3503,8 @@ namespace libtorrent m_disconnect_started = true; #endif + if (m_disconnecting) return; + #if defined TORRENT_VERBOSE_LOGGING || defined TORRENT_ERROR_LOGGING switch (error) { @@ -3585,7 +3587,6 @@ namespace libtorrent // we cannot do this in a constructor TORRENT_ASSERT(m_in_constructor == false); if (error > 0) m_failed = true; - if (m_disconnecting) return; boost::intrusive_ptr me(this); INVARIANT_CHECK; @@ -5884,7 +5885,7 @@ namespace libtorrent } } #ifdef TORRENT_EXPENSIVE_INVARIANT_CHECKS - if (m_peer_info) + if (m_peer_info && type() == bittorrent_connection) { policy::const_iterator i = t->get_policy().begin_peer(); policy::const_iterator end = t->get_policy().end_peer(); diff --git a/src/policy.cpp b/src/policy.cpp index 8e79712d1..0e0002356 100644 --- a/src/policy.cpp +++ b/src/policy.cpp @@ -590,6 +590,7 @@ namespace libtorrent { if (p.connection || p.banned + || p.web_seed || !p.connectable || (p.seed && finished) || int(p.failcount) >= m_torrent->settings().max_failcount) @@ -1061,6 +1062,7 @@ namespace libtorrent if (m_num_connect_candidates < 0) m_num_connect_candidates = 0; } + if (p->web_seed) return; if (s) ++m_num_seeds; else --m_num_seeds; TORRENT_ASSERT(m_num_seeds >= 0); @@ -1408,15 +1410,18 @@ namespace libtorrent peer* p = c.peer_info_struct(); - TORRENT_ASSERT((std::find_if( - m_peers.begin() - , m_peers.end() - , match_peer_connection(c)) - != m_peers.end()) == (p != 0)); - // if we couldn't find the connection in our list, just ignore it. if (p == 0) return; + // web seeds are special, they're not connected via the peer list + // so they're not kept in m_peers + TORRENT_ASSERT(p->web_seed + || std::find_if( + m_peers.begin() + , m_peers.end() + , match_peer_connection(c)) + != m_peers.end()); + TORRENT_ASSERT(p->connection == &c); TORRENT_ASSERT(!is_connect_candidate(*p, m_finished)); @@ -1616,6 +1621,9 @@ namespace libtorrent policy::peer* p = static_cast(*i); if (p == 0) continue; if (p->connection == 0) continue; + // web seeds are special, they're not connected via the peer list + // so they're not kept in m_peers + if (p->connection->type() != peer_connection::bittorrent_connection) continue; TORRENT_ASSERT(std::find_if(m_peers.begin(), m_peers.end() , match_peer_connection_or_endpoint(*p->connection)) != m_peers.end()); } @@ -1677,6 +1685,7 @@ namespace libtorrent , supports_utp(true) // assume peers support utp , confirmed_supports_utp(false) , supports_holepunch(false) + , web_seed(false) #if defined TORRENT_DEBUG || TORRENT_RELEASE_ASSERTS , in_use(false) #endif diff --git a/src/session_impl.cpp b/src/session_impl.cpp index 89738c263..38cdf311b 100644 --- a/src/session_impl.cpp +++ b/src/session_impl.cpp @@ -6093,7 +6093,7 @@ namespace aux { ++num_optimistic; TORRENT_ASSERT(!p->is_choked()); } - if (t && p->peer_info_struct()) + if (t && p->peer_info_struct() && !p->peer_info_struct()->web_seed) { TORRENT_ASSERT(t->get_policy().has_connection(p)); } diff --git a/src/torrent.cpp b/src/torrent.cpp index ba176e8d8..6ba342283 100644 --- a/src/torrent.cpp +++ b/src/torrent.cpp @@ -4353,6 +4353,22 @@ namespace libtorrent remove_web_seed(web); return; } + + if (web->peer_info.banned) + { +#if defined TORRENT_VERBOSE_LOGGING || defined TORRENT_LOGGING + (*m_ses.m_logger) << time_now_string() << "banned web seed: " << web->url << "\n"; +#endif + if (m_ses.m_alerts.should_post()) + { + m_ses.m_alerts.post_alert( + url_seed_alert(get_handle(), web->url + , error_code(libtorrent::errors::peer_banned, get_libtorrent_category()))); + } + // never try it again + remove_web_seed(web); + return; + } #ifdef TORRENT_USE_OPENSSL if (protocol != "http" && protocol != "https") @@ -4575,7 +4591,7 @@ namespace libtorrent } TORRENT_ASSERT(web->resolving == false); - TORRENT_ASSERT(web->connection == 0); + TORRENT_ASSERT(web->peer_info.connection == 0); web->endpoint = a; @@ -4661,13 +4677,13 @@ namespace libtorrent if (web->type == web_seed_entry::url_seed) { c = new (std::nothrow) web_peer_connection( - m_ses, shared_from_this(), s, a, web->url, 0, // TODO: pass in web + m_ses, shared_from_this(), s, a, web->url, &web->peer_info, // TODO: pass in web web->auth, web->extra_headers); } else if (web->type == web_seed_entry::http_seed) { c = new (std::nothrow) http_seed_connection( - m_ses, shared_from_this(), s, a, web->url, 0, // TODO: pass in web + m_ses, shared_from_this(), s, a, web->url, &web->peer_info, // TODO: pass in web web->auth, web->extra_headers); } if (!c) return; @@ -4692,9 +4708,16 @@ namespace libtorrent m_connections.insert(boost::get_pointer(c)); m_ses.m_connections.insert(c); - TORRENT_ASSERT(!web->connection); - web->connection = c.get(); + TORRENT_ASSERT(!web->peer_info.connection); + web->peer_info.connection = c.get(); +#if defined TORRENT_DEBUG || TORRENT_RELEASE_ASSERTS + web->peer_info.in_use = true; +#endif + c->add_stat(size_type(web->peer_info.prev_amount_download) << 10 + , size_type(web->peer_info.prev_amount_upload) << 10); + web->peer_info.prev_amount_download = 0; + web->peer_info.prev_amount_upload = 0; #if defined TORRENT_VERBOSE_LOGGING (*m_ses.m_logger) << time_now_string() << " web seed connection started " << web->url << "\n"; #endif @@ -7355,7 +7378,7 @@ namespace libtorrent i != m_web_seeds.end();) { std::list::iterator w = i++; - if (w->connection) continue; + if (w->peer_info.connection) continue; if (w->retry > time_now()) continue; if (w->resolving) continue; @@ -7808,11 +7831,46 @@ namespace libtorrent return ret; } + void torrent::remove_web_seed(std::string const& url, web_seed_entry::type_t type) + { + std::list::iterator i = std::find_if(m_web_seeds.begin(), m_web_seeds.end() + , (boost::bind(&web_seed_entry::url, _1) + == url && boost::bind(&web_seed_entry::type, _1) == type)); + if (i != m_web_seeds.end()) remove_web_seed(i); + } + + void torrent::disconnect_web_seed(peer_connection* p) + { + std::list::iterator i = std::find_if(m_web_seeds.begin(), m_web_seeds.end() + , (boost::bind(&policy::peer::connection, boost::bind(&web_seed_entry::peer_info, _1)) == p)); + // this happens if the web server responded with a redirect + // or with something incorrect, so that we removed the web seed + // immediately, before we disconnected + if (i == m_web_seeds.end()) return; + + TORRENT_ASSERT(i->resolving == false); + +#if defined TORRENT_VERBOSE_LOGGING || defined TORRENT_ERROR_LOGGING + (*m_ses.m_logger) << time_now_string() << " disconnect_web_seed: " << i->url << "\n"; +#endif + TORRENT_ASSERT(i->peer_info.connection); + i->peer_info.connection = 0; + } + + void torrent::remove_web_seed(peer_connection* p) + { + std::list::iterator i = std::find_if(m_web_seeds.begin(), m_web_seeds.end() + , (boost::bind(&policy::peer::connection, boost::bind(&web_seed_entry::peer_info, _1)) == p)); + TORRENT_ASSERT(i != m_web_seeds.end()); + if (i == m_web_seeds.end()) return; + m_web_seeds.erase(i); + } + void torrent::retry_web_seed(peer_connection* p, int retry) { TORRENT_ASSERT(m_ses.is_network_thread()); std::list::iterator i = std::find_if(m_web_seeds.begin(), m_web_seeds.end() - , (boost::bind(&web_seed_entry::connection, _1) == p)); + , (boost::bind(&policy::peer::connection, boost::bind(&web_seed_entry::peer_info, _1)) == p)); TORRENT_ASSERT(i != m_web_seeds.end()); if (i == m_web_seeds.end()) return; diff --git a/src/torrent_info.cpp b/src/torrent_info.cpp index 9d934de50..fb31b0c24 100644 --- a/src/torrent_info.cpp +++ b/src/torrent_info.cpp @@ -498,6 +498,17 @@ namespace libtorrent url.erase(url.begin()); } + web_seed_entry::web_seed_entry(std::string const& url_, type_t type_ + , std::string const& auth_ + , headers_t const& extra_headers_) + : url(url_), type(type_) + , auth(auth_), extra_headers(extra_headers_) + , retry(time_now()), resolving(false), removed(false) + , peer_info(0, true, 0) + { + peer_info.web_seed = true; + } + torrent_info::torrent_info(torrent_info const& t, int flags) : m_merkle_first_leaf(t.m_merkle_first_leaf) , m_files(t.m_files) diff --git a/test/test_web_seed.cpp b/test/test_web_seed.cpp index 2123202a9..381558551 100644 --- a/test/test_web_seed.cpp +++ b/test/test_web_seed.cpp @@ -48,7 +48,7 @@ using namespace libtorrent; // proxy: 0=none, 1=socks4, 2=socks5, 3=socks5_pw 4=http 5=http_pw void test_transfer(boost::intrusive_ptr torrent_file - , int proxy, int port, char const* protocol, bool url_seed, bool chunked_encoding) + , int proxy, int port, char const* protocol, bool url_seed, bool chunked_encoding, bool test_ban) { using namespace libtorrent; @@ -65,8 +65,8 @@ void test_transfer(boost::intrusive_ptr torrent_file char const* test_name[] = {"no", "SOCKS4", "SOCKS5", "SOCKS5 password", "HTTP", "HTTP password"}; - fprintf(stderr, "\n\n ==== TESTING === proxy: %s ==== protocol: %s ==== seed: %s === transfer-encoding: %s\n\n\n" - , test_name[proxy], protocol, url_seed ? "URL seed" : "HTTP seed", chunked_encoding ? "chunked": "none"); + fprintf(stderr, "\n\n ==== TESTING === proxy: %s ==== protocol: %s ==== seed: %s === transfer-encoding: %s === corruption: %s\n\n\n" + , test_name[proxy], protocol, url_seed ? "URL seed" : "HTTP seed", chunked_encoding ? "chunked": "none", test_ban ? "yes" : "no"); if (proxy) { @@ -128,7 +128,13 @@ void test_transfer(boost::intrusive_ptr torrent_file << " buffers: " << cs.total_used_buffers << std::endl; */ - print_alerts(ses, " >> ses", false, false, false, 0, true); + print_alerts(ses, " >> ses", test_ban, false, false, 0, true); + + if (test_ban && th.url_seeds().empty()) + { + // when we don't have any web seeds left, we know we successfully banned it + break; + } if (s.is_seeding /* && ss.download_rate == 0.f*/) { @@ -142,6 +148,10 @@ void test_transfer(boost::intrusive_ptr torrent_file test_sleep(500); } + // for test_ban tests, make sure we removed + // the url seed (i.e. banned it) + TEST_CHECK(!test_ban || th.url_seeds().empty()); + TEST_EQUAL(cs.cache_size, 0); TEST_EQUAL(cs.total_used_buffers, 0); @@ -157,12 +167,14 @@ void test_transfer(boost::intrusive_ptr torrent_file // TEST_CHECK(fabs(rate_sum - total_size) < total_size * .1f); // TEST_CHECK(fabs(ses_rate_sum - total_size) < total_size * .1f); - TEST_CHECK(th.status().is_seeding); + // if test_ban is true, we're not supposed to have completed the download + // otherwise, we are supposed to have + TEST_CHECK(th.status().is_seeding == !test_ban); if (proxy) stop_proxy(8002); TEST_CHECK(exists(combine_path("tmp2_web_seed", torrent_file->files().file_path( - torrent_file->file_at(0))))); + torrent_file->file_at(0)))) || test_ban); remove_all("tmp2_web_seed", ec); } @@ -198,7 +210,7 @@ sha1_hash file_hash(std::string const& name) } // test_url_seed determines whether to use url-seed or http-seed -int run_suite(char const* protocol, bool test_url_seed, bool chunked_encoding) +int run_suite(char const* protocol, bool test_url_seed, bool chunked_encoding, bool test_ban) { using namespace libtorrent; @@ -208,12 +220,12 @@ int run_suite(char const* protocol, bool test_url_seed, bool chunked_encoding) file_storage fs; std::srand(10); int piece_size = 0x4000; + static const int file_sizes[] = + { 5, 16 - 5, 16000, 17, 10, 8000, 8000, 1,1,1,1,1,100,1,1,1,1,100,1,1,1,1,1,1 + ,1,1,1,1,1,1,13,65000,34,75,2,30,400,500,23000,900,43000,400,4300,6, 4}; + if (test_url_seed) { - int file_sizes[] = - { 5, 16 - 5, 16000, 17, 10, 8000, 8000, 1,1,1,1,1,100,1,1,1,1,100,1,1,1,1,1,1 - ,1,1,1,1,1,1,13,65000,34,75,2,30,400,500,23000,900,43000,400,4300,6, 4}; - char* random_data = (char*)malloc(300000); for (int i = 0; i != sizeof(file_sizes)/sizeof(file_sizes[0]); ++i) { @@ -252,6 +264,7 @@ int run_suite(char const* protocol, bool test_url_seed, bool chunked_encoding) // are not requested web seeds libtorrent::create_torrent t(fs, piece_size, 0x4000, libtorrent::create_torrent::optimize | libtorrent::create_torrent::calculate_file_hashes); + char tmp[512]; if (test_url_seed) { @@ -284,29 +297,68 @@ int run_suite(char const* protocol, bool test_url_seed, bool chunked_encoding) return 0; } + if (test_ban) + { + // corrupt the files now, so that the web seed will be banned + if (test_url_seed) + { + char* random_data = (char*)malloc(300000); + for (int i = 0; i != sizeof(file_sizes)/sizeof(file_sizes[0]); ++i) + { + std::generate(random_data, random_data + 300000, &std::rand); + char filename[200]; + snprintf(filename, sizeof(filename), "tmp1_web_seed/test_torrent_dir/test%d", i); + int to_write = file_sizes[i]; + file f(filename, file::write_only, ec); + size_type offset = 0; + while (to_write > 0) + { + int s = (std::min)(to_write, 300000); + file::iovec_t b = { random_data, s}; + f.writev(offset, &b, 1, ec); + offset += s; + to_write -= s; + } + } + free(random_data); + } + else + { + piece_size = 64 * 1024; + char* random_data = (char*)malloc(64 * 1024 * 25); + std::generate(random_data, random_data + 64 * 1024 * 25, &std::rand); + save_file("tmp1_web_seed/seed", random_data, 64 * 1024 * 25); + free(random_data); + } + } + std::vector buf; bencode(std::back_inserter(buf), t.generate()); boost::intrusive_ptr torrent_file(new torrent_info(&buf[0], buf.size(), ec)); - // verify that the file hashes are correct - for (int i = 0; i < torrent_file->num_files(); ++i) + // no point in testing the hashes since we know the data is corrupt + if (!test_ban) { - sha1_hash h1 = torrent_file->file_at(i).filehash; - sha1_hash h2 = file_hash(combine_path("tmp1_web_seed" - , torrent_file->file_at(i).path)); -// fprintf(stderr, "%s: %s == %s\n" -// , torrent_file->file_at(i).path.c_str() -// , to_hex(h1.to_string()).c_str(), to_hex(h2.to_string()).c_str()); - TEST_EQUAL(h1, h2); + // verify that the file hashes are correct + for (int i = 0; i < torrent_file->num_files(); ++i) + { + sha1_hash h1 = torrent_file->file_at(i).filehash; + sha1_hash h2 = file_hash(combine_path("tmp1_web_seed" + , torrent_file->file_at(i).path)); +// fprintf(stderr, "%s: %s == %s\n" +// , torrent_file->file_at(i).path.c_str() +// , to_hex(h1.to_string()).c_str(), to_hex(h2.to_string()).c_str()); + TEST_EQUAL(h1, h2); + } } for (int i = 0; i < 6; ++i) - test_transfer(torrent_file, i, port, protocol, test_url_seed, chunked_encoding); + test_transfer(torrent_file, i, port, protocol, test_url_seed, chunked_encoding, test_ban); if (test_url_seed) { torrent_file->rename_file(0, "tmp2_web_seed/test_torrent_dir/renamed_test1"); - test_transfer(torrent_file, 0, port, protocol, test_url_seed, chunked_encoding); + test_transfer(torrent_file, 0, port, protocol, test_url_seed, chunked_encoding, test_ban); } stop_web_server(); @@ -317,14 +369,17 @@ int run_suite(char const* protocol, bool test_url_seed, bool chunked_encoding) int test_main() { int ret = 0; - for (int i = 0; i < 2; ++i) + for (int url_seed = 0; url_seed < 2; ++url_seed) { - for (int j = 0; j < 2; ++j) + for (int chunked = 0; chunked < 2; ++chunked) { + for (int ban = 0; ban < 2; ++ban) + { #ifdef TORRENT_USE_OPENSSL - run_suite("https", i, j); + run_suite("https", url_seed, chunked, ban); #endif - run_suite("http", i, j); + run_suite("http", url_seed, chunked, ban); + } } } return ret;