From 961417814b73029603c7e73ac2013eb69580a575 Mon Sep 17 00:00:00 2001 From: arvidn Date: Mon, 9 Sep 2019 14:34:31 +0200 Subject: [PATCH] WIP: fix to be less likely to hammer web seeds --- include/libtorrent/torrent.hpp | 6 ++++ src/torrent.cpp | 4 +-- src/web_peer_connection.cpp | 61 ++++++++++++++++++++++++++++------ 3 files changed, 59 insertions(+), 12 deletions(-) diff --git a/include/libtorrent/torrent.hpp b/include/libtorrent/torrent.hpp index d02e3b80f..6d4b9c2eb 100644 --- a/include/libtorrent/torrent.hpp +++ b/include/libtorrent/torrent.hpp @@ -173,6 +173,12 @@ namespace libtorrent { // callback remove it bool removed = false; + // this indicates whether this web seed has any files. A server that only + // redirects to other servers for instance, may not have any files and + // once we've seen all redirects, there's no point in connecting to it + // again. + bool interesting = true; + // if this is true, this URL was created by a redirect and should not be // saved in the resume data bool ephemeral = false; diff --git a/src/torrent.cpp b/src/torrent.cpp index fee99a3bb..db319d0fb 100644 --- a/src/torrent.cpp +++ b/src/torrent.cpp @@ -9182,14 +9182,14 @@ bool is_downloading_state(int const st) int limit = zero_or(settings().get_int(settings_pack::max_web_seed_connections) , 100); - auto const now = aux::time_now(); + auto const now = aux::time_now32(); // keep trying web-seeds if there are any // first find out which web seeds we are connected to for (auto i = m_web_seeds.begin(); i != m_web_seeds.end() && limit > 0;) { auto const w = i++; - if (w->removed || w->retry > now) + if (w->removed || w->retry > now || !w->interesting) continue; --limit; diff --git a/src/web_peer_connection.cpp b/src/web_peer_connection.cpp index 2f5de1962..673ee292e 100644 --- a/src/web_peer_connection.cpp +++ b/src/web_peer_connection.cpp @@ -139,6 +139,14 @@ void web_peer_connection::on_connected() { incoming_have_all(); } + else if (m_web->have_files.none_set()) + { + incoming_have_none(); + m_web->interesting = false; +#ifndef TORRENT_DISABLE_LOGGING + peer_log(peer_log_alert::info, "WEB-SEED", "have no files, not interesting. %s", m_url.c_str()); +#endif + } else { std::shared_ptr t = associated_torrent().lock(); @@ -162,7 +170,18 @@ void web_peer_connection::on_connected() for (piece_index_t k = std::get<0>(range); k < std::get<1>(range); ++k) have.clear_bit(k); } - incoming_bitfield(have); + if (have.none_set()) + { + incoming_have_none(); + m_web->interesting = false; +#ifndef TORRENT_DISABLE_LOGGING + peer_log(peer_log_alert::info, "WEB-SEED", "have no pieces, not interesting. %s", m_url.c_str()); +#endif + } + else + { + incoming_bitfield(have); + } } // TODO: 3 this should be an optional, piece index -1 should @@ -211,6 +230,22 @@ void web_peer_connection::disconnect(error_code const& ec m_web->endpoints.erase(m_web->endpoints.begin()); } + if (ec == errors::uninteresting_upload_peer && m_web) + { + // if this is an "ephemeral" web seed, it means it was added by receiving + // an HTTP redirect. If we disconnect because we're not interested in any + // of its pieces, mark it as uninteresting, to avoid reconnecting to it + // repeatedly + if (m_web->ephemeral) m_web->interesting = false; + + // if the web seed is not ephemeral, but we're still not interested. That + // implies that all files either have failed with 404 or with a + // redirection to a different web server. + m_web->retry = std::max(m_web->retry, aux::time_now32() + + seconds32(m_settings.get_int(settings_pack::urlseed_wait_retry))); + TORRENT_ASSERT(m_web->retry > aux::time_now32()); + } + std::shared_ptr t = associated_torrent().lock(); if (!m_requests.empty() && !m_file_requests.empty() @@ -248,6 +283,13 @@ void web_peer_connection::disconnect(error_code const& ec if (t) get_io_service().post( std::bind(&torrent::maybe_connect_web_seeds, t)); } + + if (error >= failure) + { + m_web->retry = std::max(m_web->retry, aux::time_now32() + + seconds32(m_settings.get_int(settings_pack::urlseed_wait_retry))); + } + peer_connection::disconnect(ec, op, error); if (t) t->disconnect_web_seed(this); } @@ -400,9 +442,6 @@ void web_peer_connection::write_request(peer_request const& r) continue; } - TORRENT_ASSERT(m_web->have_files.empty() - || m_web->have_files.get_bit(f.file_index)); - request += "GET "; if (using_proxy) { @@ -577,10 +616,9 @@ void web_peer_connection::handle_error(int const bytes_left) // associated with the file we just requested. Only // when it doesn't have any of the file do the following // pad files will make it complicated - auto const retry_time = value_or(m_parser.header_duration("retry-after") - , seconds32(m_settings.get_int(settings_pack::urlseed_wait_retry))); + // temporarily unavailable, retry later - t->retry_web_seed(this, retry_time); + t->retry_web_seed(this, m_parser.header_duration("retry-after")); if (t->alerts().should_post()) { std::string const error_msg = to_string(m_parser.status_code()).data() @@ -668,10 +706,13 @@ void web_peer_connection::handle_redirect(int const bytes_left) // connected to it. Make it advertise that it has this file to the // bittorrent engine file_storage const& fs = t->torrent_file().files(); - auto const range = aux::file_piece_range_exclusive(fs, file_index); + auto const range = aux::file_piece_range_inclusive(fs, file_index); for (piece_index_t i = std::get<0>(range); i < std::get<1>(range); ++i) pc->incoming_have(i); } + // we just learned about another file this web server has, make sure + // it's marked interesting to enable connecting to it + web->interesting = true; } // we don't have this file on this server. Don't ask for it again @@ -683,8 +724,8 @@ void web_peer_connection::handle_redirect(int const bytes_left) peer_log(peer_log_alert::info, "MISSING_FILE", "redirection | file: %d" , static_cast(file_index)); #endif - disconnect(errors::redirecting, operation_t::bittorrent, peer_error); } + disconnect(errors::redirecting, operation_t::bittorrent, normal); } else { @@ -698,7 +739,7 @@ void web_peer_connection::handle_redirect(int const bytes_left) // this web seed doesn't have any files. Don't try to request from it // again this session m_web->have_files.resize(t->torrent_file().num_files(), false); - disconnect(errors::redirecting, operation_t::bittorrent, peer_error); + disconnect(errors::redirecting, operation_t::bittorrent, normal); m_web = nullptr; TORRENT_ASSERT(is_disconnecting()); }