diff --git a/ChangeLog b/ChangeLog index 077b80cd9..634d0eecc 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,4 @@ + * improved support for web seeds that don't support keep-alive * improve DHT routing table to return better nodes (lower RTT and closer to target) * don't use pointers to resume_data and file_priorities in add_torrent_params * allow moving files to absolute paths, out of the download directory diff --git a/include/libtorrent/http_parser.hpp b/include/libtorrent/http_parser.hpp index 449cfbcd5..53023f01f 100644 --- a/include/libtorrent/http_parser.hpp +++ b/include/libtorrent/http_parser.hpp @@ -124,6 +124,8 @@ namespace libtorrent // reset the whole state and start over void reset(); + bool connection_close() const { return m_connection_close; } + std::multimap const& headers() const { return m_header; } std::vector > const& chunks() const { return m_chunked_ranges; } @@ -145,6 +147,9 @@ namespace libtorrent buffer::const_interval m_recv_buffer; int m_body_start_pos; + // this is true if the server is HTTP/1.0 or + // if it sent "connection: close" + bool m_connection_close; bool m_chunked_encoding; bool m_finished; diff --git a/include/libtorrent/http_seed_connection.hpp b/include/libtorrent/http_seed_connection.hpp index b3cadb61b..152960db5 100644 --- a/include/libtorrent/http_seed_connection.hpp +++ b/include/libtorrent/http_seed_connection.hpp @@ -86,10 +86,7 @@ namespace libtorrent , boost::weak_ptr t , boost::shared_ptr s , tcp::endpoint const& remote - , std::string const& url - , policy::peer* peerinfo - , std::string const& ext_auth - , web_seed_entry::headers_t const& ext_headers); + , web_seed_entry& web); virtual int type() const { return peer_connection::http_seed_connection; } diff --git a/include/libtorrent/torrent_info.hpp b/include/libtorrent/torrent_info.hpp index eb901dc1e..0f51f8ab9 100644 --- a/include/libtorrent/torrent_info.hpp +++ b/include/libtorrent/torrent_info.hpp @@ -263,6 +263,11 @@ namespace libtorrent // if this is > now, we can't reconnect yet ptime retry; + // this is initialized to true, but if we discover the + // server not to support it, it's set to false, and we + // make larger requests. + bool supports_keepalive; + // this indicates whether or not we're resolving the // hostname of this URL bool resolving; diff --git a/include/libtorrent/web_connection_base.hpp b/include/libtorrent/web_connection_base.hpp index 9af0d9d0f..07c84eb00 100644 --- a/include/libtorrent/web_connection_base.hpp +++ b/include/libtorrent/web_connection_base.hpp @@ -95,10 +95,7 @@ namespace libtorrent , boost::weak_ptr t , boost::shared_ptr s , tcp::endpoint const& remote - , std::string const& url - , policy::peer* peerinfo - , std::string const& ext_auth - , web_seed_entry::headers_t const& ext_headers); + , web_seed_entry& web); void start(); ~web_connection_base(); diff --git a/include/libtorrent/web_peer_connection.hpp b/include/libtorrent/web_peer_connection.hpp index 647eecbf4..c1b283284 100644 --- a/include/libtorrent/web_peer_connection.hpp +++ b/include/libtorrent/web_peer_connection.hpp @@ -84,10 +84,7 @@ namespace libtorrent , boost::weak_ptr t , boost::shared_ptr s , tcp::endpoint const& remote - , std::string const& url - , policy::peer* peerinfo - , std::string const& ext_auth - , web_seed_entry::headers_t const& ext_headers); + , web_seed_entry& web); virtual int type() const { return peer_connection::url_seed_connection; } @@ -121,6 +118,8 @@ namespace libtorrent std::deque m_file_requests; std::string m_url; + + web_seed_entry& m_web; // this is used for intermediate storage of pieces // that are received in more than one HTTP response @@ -151,6 +150,10 @@ namespace libtorrent // this is the number of bytes we've already received // from the next chunk header we're waiting for int m_partial_chunk_header; + + // the number of responses we've received so far on + // this connection + int m_num_responses; }; } diff --git a/src/http_parser.cpp b/src/http_parser.cpp index 1c48d78f7..9d8606d9e 100644 --- a/src/http_parser.cpp +++ b/src/http_parser.cpp @@ -133,6 +133,10 @@ restart_response: { m_status_code = atoi(read_until(line, ' ', line_end).c_str()); m_server_message = read_until(line, '\r', line_end); + + // HTTP 1.0 always closes the connection after + // each request + if (m_protocol == "HTTP/1.0") m_connection_close = true; } else { @@ -203,6 +207,10 @@ restart_response: { m_content_length = strtoll(value.c_str(), 0, 10); } + else if (name == "connection") + { + m_connection_close = string_begins_no_case("close", value.c_str()); + } else if (name == "content-range") { bool success = true; diff --git a/src/http_seed_connection.cpp b/src/http_seed_connection.cpp index 0fd86fedc..90d1a19b8 100644 --- a/src/http_seed_connection.cpp +++ b/src/http_seed_connection.cpp @@ -59,12 +59,9 @@ namespace libtorrent , boost::weak_ptr t , boost::shared_ptr s , tcp::endpoint const& remote - , std::string const& url - , policy::peer* peerinfo - , std::string const& auth - , web_seed_entry::headers_t const& extra_headers) - : web_connection_base(ses, t, s, remote, url, peerinfo, auth, extra_headers) - , m_url(url) + , web_seed_entry& web) + : web_connection_base(ses, t, s, remote, web) + , m_url(web.url) , m_response_left(0) , m_chunk_pos(0) , m_partial_chunk_header(0) diff --git a/src/peer_connection.cpp b/src/peer_connection.cpp index 8944b5ef1..796c5af49 100644 --- a/src/peer_connection.cpp +++ b/src/peer_connection.cpp @@ -811,7 +811,7 @@ namespace libtorrent if (t->is_sequential_download()) { - ret |= piece_picker::sequential | piece_picker::ignore_whole_pieces; + ret |= piece_picker::sequential; } else if (t->num_have() < t->settings().initial_picker_threshold) { diff --git a/src/torrent.cpp b/src/torrent.cpp index 975aa7552..c38f8b69c 100644 --- a/src/torrent.cpp +++ b/src/torrent.cpp @@ -4924,14 +4924,12 @@ namespace libtorrent if (web->type == web_seed_entry::url_seed) { c = new (std::nothrow) web_peer_connection( - m_ses, shared_from_this(), s, a, web->url, &web->peer_info, - web->auth, web->extra_headers); + m_ses, shared_from_this(), s, a, *web); } else if (web->type == web_seed_entry::http_seed) { c = new (std::nothrow) http_seed_connection( - m_ses, shared_from_this(), s, a, web->url, &web->peer_info, - web->auth, web->extra_headers); + m_ses, shared_from_this(), s, a, *web); } if (!c) return; diff --git a/src/torrent_info.cpp b/src/torrent_info.cpp index 13576f526..078dddaa5 100644 --- a/src/torrent_info.cpp +++ b/src/torrent_info.cpp @@ -555,7 +555,9 @@ namespace libtorrent , headers_t const& extra_headers_) : url(url_), type(type_) , auth(auth_), extra_headers(extra_headers_) - , retry(time_now()), resolving(false), removed(false) + , retry(time_now()) + , supports_keepalive(true) + , resolving(false), removed(false) , peer_info(tcp::endpoint(), true, 0) { peer_info.web_seed = true; diff --git a/src/upnp.cpp b/src/upnp.cpp index d560cfc09..9bb6f28b4 100644 --- a/src/upnp.cpp +++ b/src/upnp.cpp @@ -1416,8 +1416,11 @@ void upnp::on_upnp_unmap_response(error_code const& e } error_code_parse_state s; - xml_parse((char*)p.get_body().begin, (char*)p.get_body().end - , boost::bind(&find_error_code, _1, _2, boost::ref(s))); + if (p.header_finished()) + { + xml_parse((char*)p.get_body().begin, (char*)p.get_body().end + , boost::bind(&find_error_code, _1, _2, boost::ref(s))); + } l.unlock(); m_callback(mapping, address(), 0, p.status_code() != 200 diff --git a/src/web_connection_base.cpp b/src/web_connection_base.cpp index a554034b6..580df150e 100644 --- a/src/web_connection_base.cpp +++ b/src/web_connection_base.cpp @@ -60,14 +60,11 @@ namespace libtorrent , boost::weak_ptr t , boost::shared_ptr s , tcp::endpoint const& remote - , std::string const& url - , policy::peer* peerinfo - , std::string const& auth - , web_seed_entry::headers_t const& extra_headers) - : peer_connection(ses, t, s, remote, peerinfo) + , web_seed_entry& web) + : peer_connection(ses, t, s, remote, &web.peer_info) , m_parser(http_parser::dont_parse_chunks) - , m_external_auth(auth) - , m_extra_headers(extra_headers) + , m_external_auth(web.auth) + , m_extra_headers(web.extra_headers) , m_first_request(true) , m_ssl(false) , m_body_start(0) @@ -84,7 +81,7 @@ namespace libtorrent std::string protocol; error_code ec; boost::tie(protocol, m_basic_auth, m_host, m_port, m_path) - = parse_url_components(url, ec); + = parse_url_components(web.url, ec); TORRENT_ASSERT(!ec); if (m_port == -1 && protocol == "http") diff --git a/src/web_peer_connection.cpp b/src/web_peer_connection.cpp index a36dabd30..4ef0f1990 100644 --- a/src/web_peer_connection.cpp +++ b/src/web_peer_connection.cpp @@ -65,17 +65,16 @@ namespace libtorrent , boost::weak_ptr t , boost::shared_ptr s , tcp::endpoint const& remote - , std::string const& url - , policy::peer* peerinfo - , std::string const& auth - , web_seed_entry::headers_t const& extra_headers) - : web_connection_base(ses, t, s, remote, url, peerinfo, auth, extra_headers) - , m_url(url) + , web_seed_entry& web) + : web_connection_base(ses, t, s, remote, web) + , m_url(web.url) + , m_web(web) , m_received_body(0) , m_range_pos(0) , m_block_pos(0) , m_chunk_pos(0) , m_partial_chunk_header(0) + , m_num_responses(0) { INVARIANT_CHECK; @@ -88,7 +87,13 @@ namespace libtorrent // we always prefer downloading 1 MiB chunks // from web seeds, or whole pieces if pieces // are larger than a MiB - prefer_whole_pieces((std::max)((1024 * 1024) / tor->torrent_file().piece_length(), 1)); + int preferred_size = 1024 * 1024; + + // if the web server is known not to support keep-alive. + // request even larger blocks at a time + if (!web.supports_keepalive) preferred_size *= 4; + + prefer_whole_pieces((std::max)(preferred_size / tor->torrent_file().piece_length(), 1)); // we want large blocks as well, so // we can request more bytes at once @@ -97,7 +102,7 @@ namespace libtorrent request_large_blocks(true); #ifdef TORRENT_VERBOSE_LOGGING - peer_log("*** web_peer_connection %s", url.c_str()); + peer_log("*** web_peer_connection %s", web.url.c_str()); #endif } @@ -466,6 +471,15 @@ namespace libtorrent // we just completed reading the header if (!header_finished) { + ++m_num_responses; + + if (m_parser.connection_close()) + { + incoming_choke(); + if (m_num_responses == 1) + m_web.supports_keepalive = false; + } + #ifdef TORRENT_VERBOSE_LOGGING peer_log("*** STATUS: %d %s", m_parser.status_code(), m_parser.message().c_str()); std::multimap const& headers = m_parser.headers(); diff --git a/test/test_http_parser.cpp b/test/test_http_parser.cpp index 6c06ca997..404c074ca 100644 --- a/test/test_http_parser.cpp +++ b/test/test_http_parser.cpp @@ -115,6 +115,56 @@ int test_main() == "http://192.168.1.1:5431/dyndev/uuid:000f-66d6-7296000099dc"); TEST_CHECK(parser.header("ext") == ""); TEST_CHECK(parser.header("date") == "Fri, 02 Jan 1970 08:10:38 GMT"); + TEST_EQUAL(parser.connection_close(), false); + + // test connection close + parser.reset(); + + TEST_CHECK(!parser.finished()); + + char const* http1_response = + "HTTP/1.0 200 OK\r\n" + "Cache-Control: max-age=180\r\n" + "DATE: Fri, 02 Jan 1970 08:10:38 GMT\r\n\r\n"; + + received = feed_bytes(parser, http1_response); + + TEST_CHECK(received == make_tuple(0, int(strlen(http1_response)), false)); + TEST_CHECK(parser.get_body().left() == 0); + TEST_CHECK(parser.header("date") == "Fri, 02 Jan 1970 08:10:38 GMT"); + TEST_EQUAL(parser.connection_close(), true); + + parser.reset(); + + TEST_CHECK(!parser.finished()); + + char const* close_response = + "HTTP/1.1 200 OK\r\n" + "Connection: close\r\n" + "DATE: Fri, 02 Jan 1970 08:10:38 GMT\r\n\r\n"; + + received = feed_bytes(parser, close_response); + + TEST_CHECK(received == make_tuple(0, int(strlen(close_response)), false)); + TEST_CHECK(parser.get_body().left() == 0); + TEST_CHECK(parser.header("date") == "Fri, 02 Jan 1970 08:10:38 GMT"); + TEST_EQUAL(parser.connection_close(), true); + + parser.reset(); + + TEST_CHECK(!parser.finished()); + + char const* keep_alive_response = + "HTTP/1.1 200 OK\r\n" + "Connection: keep-alive\r\n" + "DATE: Fri, 02 Jan 1970 08:10:38 GMT\r\n\r\n"; + + received = feed_bytes(parser, keep_alive_response); + + TEST_CHECK(received == make_tuple(0, int(strlen(keep_alive_response)), false)); + TEST_CHECK(parser.get_body().left() == 0); + TEST_CHECK(parser.header("date") == "Fri, 02 Jan 1970 08:10:38 GMT"); + TEST_EQUAL(parser.connection_close(), false); parser.reset(); TEST_CHECK(!parser.finished());