merged web seed redirect fix from RC_1_0

ChangeLog

@@ -27,6 +27,7 @@
 	* almost completely changed the storage interface (for custom storage)
 	* added support for hashing pieces in multiple threads
 
+	* fixed crash when web seeds redirect
 	* fix compiler warnings
 
 1.0 release

include/libtorrent/http_parser.hpp

@@ -62,6 +62,9 @@ namespace libtorrent
 	// return true if the status code is a redirect
 	bool is_redirect(int http_status);
 
+	std::string resolve_redirect_location(std::string referrer
+		, std::string location);
+
 	class TORRENT_EXTRA_EXPORT http_parser
 	{
 	public:

include/libtorrent/web_peer_connection.hpp

@@ -120,7 +120,7 @@ namespace libtorrent
 
 		std::string m_url;
 	
-		web_seed_entry& m_web;
+		web_seed_entry* m_web;
 			
 		// this is used for intermediate storage of pieces
 		// that are received in more than one HTTP response

src/http_connection.cpp

@@ -819,7 +819,7 @@ void http_connection::on_read(error_code const& e
 		{
 			int code = m_parser.status_code();
 
-			if (code >= 300 && code < 400)
+			if (is_redirect(code))
 			{
 				// attempt a redirect
 				std::string const& location = m_parser.header("location");
@@ -837,39 +837,14 @@ void http_connection::on_read(error_code const& e
 				// in its handler. For now, just kill the connection.
 //				async_shutdown(m_sock, shared_from_this());
 				m_sock.close(ec);
-				using boost::tuples::ignore;
-				boost::tie(ignore, ignore, ignore, ignore, ignore)
-					= parse_url_components(location, ec);
-				if (!ec)
-				{
-					get(location, m_completion_timeout, m_priority, &m_proxy, m_redirects - 1
-						, m_user_agent, m_bind_addr, m_resolve_flags
-#if TORRENT_USE_I2P
-						, m_i2p_conn
-#endif
-						);
-				}
-				else
-				{
-					// some broken web servers send out relative paths
-					// in the location header.
-					std::string url = m_url;
-					// remove the leaf filename
-					std::size_t i = url.find_last_of('/');
-					if (i != std::string::npos)
-						url.resize(i);
-					if ((url.empty() || url[url.size()-1] != '/')
-						&& (location.empty() || location[0] != '/'))
-						url += '/';
-					url += location;
 
-					get(url, m_completion_timeout, m_priority, &m_proxy, m_redirects - 1
+				std::string url = resolve_redirect_location(m_url, location);
-						, m_user_agent, m_bind_addr, m_resolve_flags
+				get(url, m_completion_timeout, m_priority, &m_proxy, m_redirects - 1
+					, m_user_agent, m_bind_addr, m_resolve_flags
 #if TORRENT_USE_I2P
-						, m_i2p_conn
+					, m_i2p_conn
 #endif
-						);
+					);
-				}
 				return;
 			}
 	

src/http_parser.cpp

@@ -38,6 +38,7 @@ POSSIBILITY OF SUCH DAMAGE.
 #include "libtorrent/http_parser.hpp"
 #include "libtorrent/assert.hpp"
 #include "libtorrent/escape_string.hpp"
+#include "libtorrent/parse_url.hpp" // for parse_url_components
 
 using namespace libtorrent;
 
@@ -58,6 +59,54 @@ namespace libtorrent
 			&& http_status < 400;
 	}
 
+	std::string resolve_redirect_location(std::string referrer
+		, std::string location)
+	{
+		if (location.empty()) return referrer;
+
+		error_code ec;
+		using boost::tuples::ignore;
+		boost::tie(ignore, ignore, ignore, ignore, ignore)
+			= parse_url_components(location, ec);
+
+		// if location is a full URL, just return it
+		if (!ec) return location;
+	
+		// otherwise it's likely to be just the path, or a relative path
+		std::string url = referrer;
+
+		if (location[0] == '/')
+		{
+			// it's an absolute path. replace the path component of
+			// referrer with location
+
+			// 8 is to skip the ur;l scheme://. We want the first slash
+			// that's part of the path.
+			std::size_t i = url.find_first_of('/', 8);
+			if (i == std::string::npos)
+				return location;
+			url.resize(i);
+			url += location;
+		}
+		else
+		{
+			// some web servers send out relative paths
+			// in the location header.
+			// remove the leaf filename
+			std::size_t i = url.find_last_of('/');
+			if (i == std::string::npos)
+				return location;
+
+			url.resize(i);
+
+			if ((url.empty() || url[url.size()-1] != '/')
+				&& (location.empty() || location[0] != '/'))
+				url += '/';
+			url += location;
+		}
+		return url;
+	}
+
 	http_parser::~http_parser() {}
 
 	http_parser::http_parser(int flags)

src/web_peer_connection.cpp

@@ -70,7 +70,7 @@ web_peer_connection::web_peer_connection(
 	, web_seed_entry& web)
 	: web_connection_base(ses, sett, allocator, disk_thread, t, s, web)
 	, m_url(web.url)
-	, m_web(web)
+	, m_web(&web)
 	, m_received_body(0)
 	, m_range_pos(0)
 	, m_chunk_pos(0)
@@ -111,11 +111,11 @@ web_peer_connection::web_peer_connection(
 void web_peer_connection::on_connected()
 {
 	incoming_have_all();
-	if (m_web.restart_request.piece != -1)
+	if (m_web->restart_request.piece != -1)
 	{
 		// increase the chances of requesting the block
 		// we have partial data for already, to finish it
-		incoming_suggest(m_web.restart_request.piece);
+		incoming_suggest(m_web->restart_request.piece);
 	}
 	web_connection_base::on_connected();
 }
@@ -127,7 +127,7 @@ void web_peer_connection::disconnect(error_code const& ec, peer_connection_inter
 	boost::shared_ptr<torrent> t = associated_torrent().lock();
 
 	if (!m_requests.empty() && !m_file_requests.empty()
-		&& !m_piece.empty())
+		&& !m_piece.empty() && m_web)
 	{
 #if 0
 		std::cerr << this << " SAVE-RESTART-DATA: data: " << m_piece.size()
@@ -135,15 +135,15 @@ void web_peer_connection::disconnect(error_code const& ec, peer_connection_inter
 			<< " off: " << m_requests.front().start
 			<< std::endl;
 #endif
-		m_web.restart_request = m_requests.front();
+		m_web->restart_request = m_requests.front();
-		if (!m_web.restart_piece.empty())
+		if (!m_web->restart_piece.empty())
 		{
 			// we're about to replace a different restart piece
 			// buffer. So it was wasted download
-			if (t) t->add_redundant_bytes(m_web.restart_piece.size()
+			if (t) t->add_redundant_bytes(m_web->restart_piece.size()
 				, torrent::piece_closing);
 		}
-		m_web.restart_piece.swap(m_piece);
+		m_web->restart_piece.swap(m_piece);
 
 		// we have to do this to not count this data as redundant. The
 		// upper layer will call downloading_piece_progress and assume
@@ -152,7 +152,7 @@ void web_peer_connection::disconnect(error_code const& ec, peer_connection_inter
 		m_block_pos = 0;
 	}
 
-	if (!m_web.supports_keepalive && error == 0)
+	if (m_web && !m_web->supports_keepalive && error == 0)
 	{
 		// if the web server doesn't support keepalive and we were
 		// disconnected as a graceful EOF, reconnect right away
@@ -250,9 +250,9 @@ void web_peer_connection::write_request(peer_request const& r)
 		pr.piece = r.piece + request_offset / piece_size;
 		m_requests.push_back(pr);
 
-		if (m_web.restart_request == m_requests.front())
+		if (m_web->restart_request == m_requests.front())
 		{
-			m_piece.swap(m_web.restart_piece);
+			m_piece.swap(m_web->restart_piece);
 			m_block_pos += m_piece.size();
 			peer_request& front = m_requests.front();
 			TORRENT_ASSERT(front.length > int(m_piece.size()));
@@ -270,7 +270,7 @@ void web_peer_connection::write_request(peer_request const& r)
 			// just to keep the accounting straight for the upper layer.
 			// it doesn't know we just re-wrote the request
 			incoming_piece_fragment(m_piece.size());
-			m_web.restart_request.piece = -1;
+			m_web->restart_request.piece = -1;
 		}
 
 #if 0
@@ -576,7 +576,7 @@ void web_peer_connection::on_receive(error_code const& error
 			{
 				incoming_choke();
 				if (m_num_responses == 1)
-					m_web.supports_keepalive = false;
+					m_web->supports_keepalive = false;
 			}
 
 #ifdef TORRENT_VERBOSE_LOGGING
@@ -623,6 +623,7 @@ void web_peer_connection::on_receive(error_code const& error
 				{
 					// we should not try this server again.
 					t->remove_web_seed(this, errors::missing_location, op_bittorrent, 2);
+					m_web = NULL;
 					TORRENT_ASSERT(is_disconnecting());
 #ifdef TORRENT_DEBUG
 					TORRENT_ASSERT(statistics().last_payload_downloaded()
@@ -660,6 +661,7 @@ void web_peer_connection::on_receive(error_code const& error
 						if (i == std::string::npos)
 						{
 							t->remove_web_seed(this, errors::invalid_redirection, op_bittorrent, 2);
+							m_web = NULL;
 							TORRENT_ASSERT(is_disconnecting());
 #ifdef TORRENT_DEBUG
 							TORRENT_ASSERT(statistics().last_payload_downloaded()
@@ -670,11 +672,17 @@ void web_peer_connection::on_receive(error_code const& error
 						}
 						location.resize(i);
 					}
+					else
+					{
+						location = resolve_redirect_location(m_url, location);
+					}
+
 #ifdef TORRENT_VERBOSE_LOGGING
 					peer_log("*** LOCATION: %s", location.c_str());
 #endif
 					t->add_web_seed(location, web_seed_entry::url_seed, m_external_auth, m_extra_headers);
 					t->remove_web_seed(this, errors::redirecting, op_bittorrent, 2);
+					m_web = NULL;
 					TORRENT_ASSERT(is_disconnecting());
 #ifdef TORRENT_DEBUG
 					TORRENT_ASSERT(statistics().last_payload_downloaded()
@@ -722,6 +730,7 @@ void web_peer_connection::on_receive(error_code const& error
 					received_bytes(0, bytes_transferred);
 					// we should not try this server again.
 					t->remove_web_seed(this, errors::invalid_range, op_bittorrent);
+					m_web = NULL;
 					TORRENT_ASSERT(is_disconnecting());
 #ifdef TORRENT_DEBUG
 					TORRENT_ASSERT(statistics().last_payload_downloaded()
@@ -742,6 +751,7 @@ void web_peer_connection::on_receive(error_code const& error
 					received_bytes(0, bytes_transferred);
 					// we should not try this server again.
 					t->remove_web_seed(this, errors::no_content_length, op_bittorrent, 2);
+					m_web = NULL;
 					TORRENT_ASSERT(is_disconnecting());
 #ifdef TORRENT_DEBUG
 					TORRENT_ASSERT(statistics().last_payload_downloaded()

test/Jamfile

@@ -137,6 +137,7 @@ test-suite libtorrent :
 	[ run test_checking.cpp ]
 	[ run test_url_seed.cpp ]
 	[ run test_web_seed.cpp ]
+	[ run test_web_seed_redirect.cpp ]
 	[ run test_web_seed_socks4.cpp ]
 	[ run test_web_seed_socks5.cpp ]
 	[ run test_web_seed_socks5_pw.cpp ]

test/test_http_parser.cpp

@@ -351,6 +351,24 @@ int test_main()
 	parse_url_components("http:", ec);
 	TEST_CHECK(ec == error_code(errors::unsupported_url_protocol));
 	ec.clear();
+
+	// test resolve_redirect_location
+
+	TEST_EQUAL(resolve_redirect_location("http://example.com/a/b", "a")
+		, "http://example.com/a/a");
+
+	TEST_EQUAL(resolve_redirect_location("http://example.com/a/b", "c/d/e/")
+		, "http://example.com/a/c/d/e/");
+
+	TEST_EQUAL(resolve_redirect_location("http://example.com/a/b", "../a")
+		, "http://example.com/a/../a");
+
+	TEST_EQUAL(resolve_redirect_location("http://example.com/a/b", "/c")
+		, "http://example.com/c");
+
+	TEST_EQUAL(resolve_redirect_location("http://example.com/a/b", "http://test.com/d")
+		, "http://test.com/d");
+
 	return 0;
 }
 

test/test_web_seed_redirect.cpp

@@ -0,0 +1,109 @@
+/*
+
+Copyright (c) 2008-2014, Arvid Norberg
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+    * Redistributions of source code must retain the above copyright
+      notice, this list of conditions and the following disclaimer.
+    * Redistributions in binary form must reproduce the above copyright
+      notice, this list of conditions and the following disclaimer in
+      the documentation and/or other materials provided with the distribution.
+    * Neither the name of the author nor the names of its
+      contributors may be used to endorse or promote products derived
+      from this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+POSSIBILITY OF SUCH DAMAGE.
+
+*/
+
+#include "test.hpp"
+#include "setup_transfer.hpp"
+#include "web_seed_suite.hpp"
+#include "libtorrent/create_torrent.hpp"
+
+using namespace libtorrent;
+
+const int proxy = libtorrent::proxy_settings::none;
+
+//static unsigned char random_byte()
+//{ return std::rand() & 0xff; }
+
+int test_main()
+{
+	using namespace libtorrent;
+
+	error_code ec;
+
+	file_storage fs;
+	int piece_size = 0x4000;
+
+	char random_data[16000];
+	std::generate(random_data, random_data + sizeof(random_data), random_byte);
+	file f("test_file", file::write_only, ec);
+	if (ec)
+	{
+		fprintf(stderr, "failed to create file \"test_file\": (%d) %s\n"
+			, ec.value(), ec.message().c_str());
+		return 1;
+	}
+	file::iovec_t b = { random_data, size_t(16000)};
+	f.writev(0, &b, 1, ec);
+	fs.add_file("test_file", 16000);
+
+	int port = start_web_server();
+
+	// generate a torrent with pad files to make sure they
+	// are not requested web seeds
+	libtorrent::create_torrent t(fs, piece_size, 0x4000);
+
+	char tmp[512];
+	snprintf(tmp, sizeof(tmp), "http://127.0.0.1:%d/redirect", port);
+	t.add_url_seed(tmp);
+
+	// calculate the hash for all pieces
+	set_piece_hashes(t, ".", ec);
+
+	if (ec)
+	{
+		fprintf(stderr, "error creating hashes for test torrent: %s\n"
+			, ec.message().c_str());
+		TEST_CHECK(false);
+		return 0;
+	}
+
+	std::vector<char> buf;
+	bencode(std::back_inserter(buf), t.generate());
+	boost::shared_ptr<torrent_info> torrent_file(new torrent_info(&buf[0]
+		, buf.size(), ec));
+
+	{
+		session ses(fingerprint("  ", 0,0,0,0), 0);
+		session_settings settings;
+		settings.max_queued_disk_bytes = 256 * 1024;
+		ses.set_settings(settings);
+		ses.set_alert_mask(~(alert::progress_notification | alert::stats_notification));
+   
+		// disable keep-alive because otherwise the test will choke on seeing
+		// the disconnect (from the redirect)
+		test_transfer(ses, torrent_file, 0, 0, "http", true, false, false, false);
+	}
+
+	stop_web_server();
+	return 0;
+}
+
+

test/web_seed_suite.cpp

@@ -79,7 +79,7 @@ static sha1_hash file_hash(std::string const& name)
 static char const* proxy_name[] = {"", "_socks4", "_socks5", "_socks5_pw", "_http", "_http_pw", "_i2p"};
 
 // proxy: 0=none, 1=socks4, 2=socks5, 3=socks5_pw 4=http 5=http_pw
-static void test_transfer(lt::session& ses, boost::shared_ptr<torrent_info> torrent_file
+void test_transfer(lt::session& ses, boost::shared_ptr<torrent_info> torrent_file
 	, int proxy, int port, char const* protocol, bool url_seed
 	, bool chunked_encoding, bool test_ban, bool keepalive)
 {

test/web_seed_suite.hpp

@@ -35,3 +35,9 @@ int EXPORT run_http_suite(int proxy, char const* protocol
 	, bool test_url_seed, bool chunked_encoding = false, bool test_ban = false
 	, bool keepalive = true, bool test_rename = false);
 
+void EXPORT test_transfer(libtorrent::session& ses
+	, boost::shared_ptr<libtorrent::torrent_info> torrent_file
+	, int proxy = 0, int port = 0, char const* protocol = "http"
+	, bool url_seed = true, bool chunked_encoding = false
+	, bool test_ban = false, bool keepalive = true);
+