improved support for web seeds that don't support keep-alive

ChangeLog

@@ -1,3 +1,4 @@
+	* improved support for web seeds that don't support keep-alive
 	* improve DHT routing table to return better nodes (lower RTT and closer to target)
 	* don't use pointers to resume_data and file_priorities in add_torrent_params
 	* allow moving files to absolute paths, out of the download directory

include/libtorrent/http_parser.hpp

@@ -124,6 +124,8 @@ namespace libtorrent
 		// reset the whole state and start over
 		void reset();
 
+		bool connection_close() const { return m_connection_close; }
+
 		std::multimap<std::string, std::string> const& headers() const { return m_header; }
 		std::vector<std::pair<size_type, size_type> > const& chunks() const { return m_chunked_ranges; }
 		
@@ -145,6 +147,9 @@ namespace libtorrent
 		buffer::const_interval m_recv_buffer;
 		int m_body_start_pos;
 
+		// this is true if the server is HTTP/1.0 or
+		// if it sent "connection: close"
+		bool m_connection_close;
 		bool m_chunked_encoding;
 		bool m_finished;
 

include/libtorrent/http_seed_connection.hpp

@@ -86,10 +86,7 @@ namespace libtorrent
 			, boost::weak_ptr<torrent> t
 			, boost::shared_ptr<socket_type> s
 			, tcp::endpoint const& remote
-			, std::string const& url
-			, policy::peer* peerinfo
-			, std::string const& ext_auth
-			, web_seed_entry::headers_t const& ext_headers);
+			, web_seed_entry& web);
 
 		virtual int type() const { return peer_connection::http_seed_connection; }
 

include/libtorrent/torrent_info.hpp

@@ -263,6 +263,11 @@ namespace libtorrent
 		// if this is > now, we can't reconnect yet
 		ptime retry;
 
+		// this is initialized to true, but if we discover the
+		// server not to support it, it's set to false, and we
+		// make larger requests.
+		bool supports_keepalive;
+
 		// this indicates whether or not we're resolving the
 		// hostname of this URL
 		bool resolving;

include/libtorrent/web_connection_base.hpp

@@ -95,10 +95,7 @@ namespace libtorrent
 			, boost::weak_ptr<torrent> t
 			, boost::shared_ptr<socket_type> s
 			, tcp::endpoint const& remote
-			, std::string const& url
-			, policy::peer* peerinfo
-			, std::string const& ext_auth
-			, web_seed_entry::headers_t const& ext_headers);
+			, web_seed_entry& web);
 		void start();
 
 		~web_connection_base();

include/libtorrent/web_peer_connection.hpp

@@ -84,10 +84,7 @@ namespace libtorrent
 			, boost::weak_ptr<torrent> t
 			, boost::shared_ptr<socket_type> s
 			, tcp::endpoint const& remote
-			, std::string const& url
-			, policy::peer* peerinfo
-			, std::string const& ext_auth
-			, web_seed_entry::headers_t const& ext_headers);
+			, web_seed_entry& web);
 
 		virtual int type() const { return peer_connection::url_seed_connection; }
 
@@ -122,6 +119,8 @@ namespace libtorrent
 
 		std::string m_url;
 	
+		web_seed_entry& m_web;
+			
 		// this is used for intermediate storage of pieces
 		// that are received in more than one HTTP response
 		// TODO: 1 if we make this be a disk_buffer_holder instead
@@ -151,6 +150,10 @@ namespace libtorrent
 		// this is the number of bytes we've already received
 		// from the next chunk header we're waiting for
 		int m_partial_chunk_header;
+
+		// the number of responses we've received so far on
+		// this connection
+		int m_num_responses;
 	};
 }
 

src/http_parser.cpp

@@ -133,6 +133,10 @@ restart_response:
 			{
 				m_status_code = atoi(read_until(line, ' ', line_end).c_str());
 				m_server_message = read_until(line, '\r', line_end);
+
+				// HTTP 1.0 always closes the connection after
+				// each request
+				if (m_protocol == "HTTP/1.0") m_connection_close = true;
 			}
 			else
 			{
@@ -203,6 +207,10 @@ restart_response:
 				{
 					m_content_length = strtoll(value.c_str(), 0, 10);
 				}
+				else if (name == "connection")
+				{
+					m_connection_close = string_begins_no_case("close", value.c_str());
+				}
 				else if (name == "content-range")
 				{
 					bool success = true;

src/http_seed_connection.cpp

@@ -59,12 +59,9 @@ namespace libtorrent
 		, boost::weak_ptr<torrent> t
 		, boost::shared_ptr<socket_type> s
 		, tcp::endpoint const& remote
-		, std::string const& url
-		, policy::peer* peerinfo
-		, std::string const& auth
-		, web_seed_entry::headers_t const& extra_headers)
-		: web_connection_base(ses, t, s, remote, url, peerinfo, auth, extra_headers)
-		, m_url(url)
+		, web_seed_entry& web)
+		: web_connection_base(ses, t, s, remote, web)
+		, m_url(web.url)
 		, m_response_left(0)
 		, m_chunk_pos(0)
 		, m_partial_chunk_header(0)

src/peer_connection.cpp

@@ -811,7 +811,7 @@ namespace libtorrent
 
 		if (t->is_sequential_download())
 		{
-			ret |= piece_picker::sequential | piece_picker::ignore_whole_pieces;
+			ret |= piece_picker::sequential;
 		}
 		else if (t->num_have() < t->settings().initial_picker_threshold)
 		{

src/torrent.cpp

@@ -4924,14 +4924,12 @@ namespace libtorrent
 		if (web->type == web_seed_entry::url_seed)
 		{
 			c = new (std::nothrow) web_peer_connection(
-				m_ses, shared_from_this(), s, a, web->url, &web->peer_info,
-				web->auth, web->extra_headers);
+				m_ses, shared_from_this(), s, a, *web);
 		}
 		else if (web->type == web_seed_entry::http_seed)
 		{
 			c = new (std::nothrow) http_seed_connection(
-				m_ses, shared_from_this(), s, a, web->url, &web->peer_info,
-				web->auth, web->extra_headers);
+				m_ses, shared_from_this(), s, a, *web);
 		}
 		if (!c) return;
 

src/torrent_info.cpp

@@ -555,7 +555,9 @@ namespace libtorrent
 		, headers_t const& extra_headers_)
 		: url(url_), type(type_)
 		, auth(auth_), extra_headers(extra_headers_)
-		, retry(time_now()), resolving(false), removed(false)
+		, retry(time_now())
+		, supports_keepalive(true)
+		, resolving(false), removed(false)
 		, peer_info(tcp::endpoint(), true, 0)
 	{
 		peer_info.web_seed = true;

src/upnp.cpp

@@ -1416,8 +1416,11 @@ void upnp::on_upnp_unmap_response(error_code const& e
 	}
 
 	error_code_parse_state s;
+	if (p.header_finished())
+	{
 		xml_parse((char*)p.get_body().begin, (char*)p.get_body().end
 			, boost::bind(&find_error_code, _1, _2, boost::ref(s)));
+	}
 
 	l.unlock();
 	m_callback(mapping, address(), 0, p.status_code() != 200

src/web_connection_base.cpp

@@ -60,14 +60,11 @@ namespace libtorrent
 		, boost::weak_ptr<torrent> t
 		, boost::shared_ptr<socket_type> s
 		, tcp::endpoint const& remote
-		, std::string const& url
-		, policy::peer* peerinfo
-		, std::string const& auth
-		, web_seed_entry::headers_t const& extra_headers)
-		: peer_connection(ses, t, s, remote, peerinfo)
+		, web_seed_entry& web)
+		: peer_connection(ses, t, s, remote, &web.peer_info)
 		, m_parser(http_parser::dont_parse_chunks)
-		, m_external_auth(auth)
-		, m_extra_headers(extra_headers)
+		, m_external_auth(web.auth)
+		, m_extra_headers(web.extra_headers)
 		, m_first_request(true)
 		, m_ssl(false)
 		, m_body_start(0)
@@ -84,7 +81,7 @@ namespace libtorrent
 		std::string protocol;
 		error_code ec;
 		boost::tie(protocol, m_basic_auth, m_host, m_port, m_path)
-			= parse_url_components(url, ec);
+			= parse_url_components(web.url, ec);
 		TORRENT_ASSERT(!ec);
 
 		if (m_port == -1 && protocol == "http")

src/web_peer_connection.cpp

@@ -65,17 +65,16 @@ namespace libtorrent
 		, boost::weak_ptr<torrent> t
 		, boost::shared_ptr<socket_type> s
 		, tcp::endpoint const& remote
-		, std::string const& url
-		, policy::peer* peerinfo
-		, std::string const& auth
-		, web_seed_entry::headers_t const& extra_headers)
-		: web_connection_base(ses, t, s, remote, url, peerinfo, auth, extra_headers)
-		, m_url(url)
+		, web_seed_entry& web)
+		: web_connection_base(ses, t, s, remote, web)
+		, m_url(web.url)
+		, m_web(web)
 		, m_received_body(0)
 		, m_range_pos(0)
 		, m_block_pos(0)
 		, m_chunk_pos(0)
 		, m_partial_chunk_header(0)
+		, m_num_responses(0)
 	{
 		INVARIANT_CHECK;
 
@@ -88,7 +87,13 @@ namespace libtorrent
 		// we always prefer downloading 1 MiB chunks
 		// from web seeds, or whole pieces if pieces
 		// are larger than a MiB
-		prefer_whole_pieces((std::max)((1024 * 1024) / tor->torrent_file().piece_length(), 1));
+		int preferred_size = 1024 * 1024;
+
+		// if the web server is known not to support keep-alive.
+		// request even larger blocks at a time
+		if (!web.supports_keepalive) preferred_size *= 4;
+
+		prefer_whole_pieces((std::max)(preferred_size / tor->torrent_file().piece_length(), 1));
 		
 		// we want large blocks as well, so
 		// we can request more bytes at once
@@ -97,7 +102,7 @@ namespace libtorrent
 		request_large_blocks(true);
 
 #ifdef TORRENT_VERBOSE_LOGGING
-		peer_log("*** web_peer_connection %s", url.c_str());
+		peer_log("*** web_peer_connection %s", web.url.c_str());
 #endif
 	}
 
@@ -466,6 +471,15 @@ namespace libtorrent
 			// we just completed reading the header
 			if (!header_finished)
 			{
+				++m_num_responses;
+
+				if (m_parser.connection_close())
+				{
+					incoming_choke();
+					if (m_num_responses == 1)
+						m_web.supports_keepalive = false;
+				}
+
 #ifdef TORRENT_VERBOSE_LOGGING
 				peer_log("*** STATUS: %d %s", m_parser.status_code(), m_parser.message().c_str());
 				std::multimap<std::string, std::string> const& headers = m_parser.headers();

test/test_http_parser.cpp

@@ -115,6 +115,56 @@ int test_main()
 		== "http://192.168.1.1:5431/dyndev/uuid:000f-66d6-7296000099dc");
 	TEST_CHECK(parser.header("ext") == "");
 	TEST_CHECK(parser.header("date") == "Fri, 02 Jan 1970 08:10:38 GMT");
+	TEST_EQUAL(parser.connection_close(), false);
+
+	// test connection close
+	parser.reset();
+
+	TEST_CHECK(!parser.finished());
+
+	char const* http1_response =
+		"HTTP/1.0 200 OK\r\n"
+		"Cache-Control: max-age=180\r\n"
+		"DATE: Fri, 02 Jan 1970 08:10:38 GMT\r\n\r\n";
+
+	received = feed_bytes(parser, http1_response);
+
+	TEST_CHECK(received == make_tuple(0, int(strlen(http1_response)), false));
+	TEST_CHECK(parser.get_body().left() == 0);
+	TEST_CHECK(parser.header("date") == "Fri, 02 Jan 1970 08:10:38 GMT");
+	TEST_EQUAL(parser.connection_close(), true);
+
+	parser.reset();
+
+	TEST_CHECK(!parser.finished());
+
+	char const* close_response =
+		"HTTP/1.1 200 OK\r\n"
+		"Connection: close\r\n"
+		"DATE: Fri, 02 Jan 1970 08:10:38 GMT\r\n\r\n";
+
+	received = feed_bytes(parser, close_response);
+
+	TEST_CHECK(received == make_tuple(0, int(strlen(close_response)), false));
+	TEST_CHECK(parser.get_body().left() == 0);
+	TEST_CHECK(parser.header("date") == "Fri, 02 Jan 1970 08:10:38 GMT");
+	TEST_EQUAL(parser.connection_close(), true);
+
+	parser.reset();
+
+	TEST_CHECK(!parser.finished());
+
+	char const* keep_alive_response =
+		"HTTP/1.1 200 OK\r\n"
+		"Connection: keep-alive\r\n"
+		"DATE: Fri, 02 Jan 1970 08:10:38 GMT\r\n\r\n";
+
+	received = feed_bytes(parser, keep_alive_response);
+
+	TEST_CHECK(received == make_tuple(0, int(strlen(keep_alive_response)), false));
+	TEST_CHECK(parser.get_body().left() == 0);
+	TEST_CHECK(parser.header("date") == "Fri, 02 Jan 1970 08:10:38 GMT");
+	TEST_EQUAL(parser.connection_close(), false);
 
 	parser.reset();
 	TEST_CHECK(!parser.finished());