From 34440224fc6a96835f4a8a7ce14360fb2dbc5356 Mon Sep 17 00:00:00 2001 From: Arvid Norberg Date: Wed, 9 Jul 2014 20:53:39 +0000 Subject: [PATCH] merged web seed redirect fix from RC_1_0 --- ChangeLog | 1 + include/libtorrent/http_parser.hpp | 3 + include/libtorrent/web_peer_connection.hpp | 2 +- src/http_connection.cpp | 37 ++----- src/http_parser.cpp | 49 +++++++++ src/web_peer_connection.cpp | 36 ++++--- test/Jamfile | 1 + test/test_http_parser.cpp | 18 ++++ test/test_web_seed_redirect.cpp | 109 +++++++++++++++++++++ test/web_seed_suite.cpp | 2 +- test/web_seed_suite.hpp | 6 ++ 11 files changed, 218 insertions(+), 46 deletions(-) create mode 100644 test/test_web_seed_redirect.cpp diff --git a/ChangeLog b/ChangeLog index 463346be8..6e31a8af7 100644 --- a/ChangeLog +++ b/ChangeLog @@ -27,6 +27,7 @@ * almost completely changed the storage interface (for custom storage) * added support for hashing pieces in multiple threads + * fixed crash when web seeds redirect * fix compiler warnings 1.0 release diff --git a/include/libtorrent/http_parser.hpp b/include/libtorrent/http_parser.hpp index 98c15e094..a48214388 100644 --- a/include/libtorrent/http_parser.hpp +++ b/include/libtorrent/http_parser.hpp @@ -62,6 +62,9 @@ namespace libtorrent // return true if the status code is a redirect bool is_redirect(int http_status); + std::string resolve_redirect_location(std::string referrer + , std::string location); + class TORRENT_EXTRA_EXPORT http_parser { public: diff --git a/include/libtorrent/web_peer_connection.hpp b/include/libtorrent/web_peer_connection.hpp index fcec076fc..e15ad85e2 100644 --- a/include/libtorrent/web_peer_connection.hpp +++ b/include/libtorrent/web_peer_connection.hpp @@ -120,7 +120,7 @@ namespace libtorrent std::string m_url; - web_seed_entry& m_web; + web_seed_entry* m_web; // this is used for intermediate storage of pieces // that are received in more than one HTTP response diff --git a/src/http_connection.cpp b/src/http_connection.cpp index a1bcf0c36..c1e23ba9b 100644 --- a/src/http_connection.cpp +++ b/src/http_connection.cpp @@ -819,7 +819,7 @@ void http_connection::on_read(error_code const& e { int code = m_parser.status_code(); - if (code >= 300 && code < 400) + if (is_redirect(code)) { // attempt a redirect std::string const& location = m_parser.header("location"); @@ -837,39 +837,14 @@ void http_connection::on_read(error_code const& e // in its handler. For now, just kill the connection. // async_shutdown(m_sock, shared_from_this()); m_sock.close(ec); - using boost::tuples::ignore; - boost::tie(ignore, ignore, ignore, ignore, ignore) - = parse_url_components(location, ec); - if (!ec) - { - get(location, m_completion_timeout, m_priority, &m_proxy, m_redirects - 1 - , m_user_agent, m_bind_addr, m_resolve_flags -#if TORRENT_USE_I2P - , m_i2p_conn -#endif - ); - } - else - { - // some broken web servers send out relative paths - // in the location header. - std::string url = m_url; - // remove the leaf filename - std::size_t i = url.find_last_of('/'); - if (i != std::string::npos) - url.resize(i); - if ((url.empty() || url[url.size()-1] != '/') - && (location.empty() || location[0] != '/')) - url += '/'; - url += location; - get(url, m_completion_timeout, m_priority, &m_proxy, m_redirects - 1 - , m_user_agent, m_bind_addr, m_resolve_flags + std::string url = resolve_redirect_location(m_url, location); + get(url, m_completion_timeout, m_priority, &m_proxy, m_redirects - 1 + , m_user_agent, m_bind_addr, m_resolve_flags #if TORRENT_USE_I2P - , m_i2p_conn + , m_i2p_conn #endif - ); - } + ); return; } diff --git a/src/http_parser.cpp b/src/http_parser.cpp index e2c5c04c6..99228e443 100644 --- a/src/http_parser.cpp +++ b/src/http_parser.cpp @@ -38,6 +38,7 @@ POSSIBILITY OF SUCH DAMAGE. #include "libtorrent/http_parser.hpp" #include "libtorrent/assert.hpp" #include "libtorrent/escape_string.hpp" +#include "libtorrent/parse_url.hpp" // for parse_url_components using namespace libtorrent; @@ -58,6 +59,54 @@ namespace libtorrent && http_status < 400; } + std::string resolve_redirect_location(std::string referrer + , std::string location) + { + if (location.empty()) return referrer; + + error_code ec; + using boost::tuples::ignore; + boost::tie(ignore, ignore, ignore, ignore, ignore) + = parse_url_components(location, ec); + + // if location is a full URL, just return it + if (!ec) return location; + + // otherwise it's likely to be just the path, or a relative path + std::string url = referrer; + + if (location[0] == '/') + { + // it's an absolute path. replace the path component of + // referrer with location + + // 8 is to skip the ur;l scheme://. We want the first slash + // that's part of the path. + std::size_t i = url.find_first_of('/', 8); + if (i == std::string::npos) + return location; + url.resize(i); + url += location; + } + else + { + // some web servers send out relative paths + // in the location header. + // remove the leaf filename + std::size_t i = url.find_last_of('/'); + if (i == std::string::npos) + return location; + + url.resize(i); + + if ((url.empty() || url[url.size()-1] != '/') + && (location.empty() || location[0] != '/')) + url += '/'; + url += location; + } + return url; + } + http_parser::~http_parser() {} http_parser::http_parser(int flags) diff --git a/src/web_peer_connection.cpp b/src/web_peer_connection.cpp index 8661a071d..a78e38e3f 100644 --- a/src/web_peer_connection.cpp +++ b/src/web_peer_connection.cpp @@ -70,7 +70,7 @@ web_peer_connection::web_peer_connection( , web_seed_entry& web) : web_connection_base(ses, sett, allocator, disk_thread, t, s, web) , m_url(web.url) - , m_web(web) + , m_web(&web) , m_received_body(0) , m_range_pos(0) , m_chunk_pos(0) @@ -111,11 +111,11 @@ web_peer_connection::web_peer_connection( void web_peer_connection::on_connected() { incoming_have_all(); - if (m_web.restart_request.piece != -1) + if (m_web->restart_request.piece != -1) { // increase the chances of requesting the block // we have partial data for already, to finish it - incoming_suggest(m_web.restart_request.piece); + incoming_suggest(m_web->restart_request.piece); } web_connection_base::on_connected(); } @@ -127,7 +127,7 @@ void web_peer_connection::disconnect(error_code const& ec, peer_connection_inter boost::shared_ptr t = associated_torrent().lock(); if (!m_requests.empty() && !m_file_requests.empty() - && !m_piece.empty()) + && !m_piece.empty() && m_web) { #if 0 std::cerr << this << " SAVE-RESTART-DATA: data: " << m_piece.size() @@ -135,15 +135,15 @@ void web_peer_connection::disconnect(error_code const& ec, peer_connection_inter << " off: " << m_requests.front().start << std::endl; #endif - m_web.restart_request = m_requests.front(); - if (!m_web.restart_piece.empty()) + m_web->restart_request = m_requests.front(); + if (!m_web->restart_piece.empty()) { // we're about to replace a different restart piece // buffer. So it was wasted download - if (t) t->add_redundant_bytes(m_web.restart_piece.size() + if (t) t->add_redundant_bytes(m_web->restart_piece.size() , torrent::piece_closing); } - m_web.restart_piece.swap(m_piece); + m_web->restart_piece.swap(m_piece); // we have to do this to not count this data as redundant. The // upper layer will call downloading_piece_progress and assume @@ -152,7 +152,7 @@ void web_peer_connection::disconnect(error_code const& ec, peer_connection_inter m_block_pos = 0; } - if (!m_web.supports_keepalive && error == 0) + if (m_web && !m_web->supports_keepalive && error == 0) { // if the web server doesn't support keepalive and we were // disconnected as a graceful EOF, reconnect right away @@ -250,9 +250,9 @@ void web_peer_connection::write_request(peer_request const& r) pr.piece = r.piece + request_offset / piece_size; m_requests.push_back(pr); - if (m_web.restart_request == m_requests.front()) + if (m_web->restart_request == m_requests.front()) { - m_piece.swap(m_web.restart_piece); + m_piece.swap(m_web->restart_piece); m_block_pos += m_piece.size(); peer_request& front = m_requests.front(); TORRENT_ASSERT(front.length > int(m_piece.size())); @@ -270,7 +270,7 @@ void web_peer_connection::write_request(peer_request const& r) // just to keep the accounting straight for the upper layer. // it doesn't know we just re-wrote the request incoming_piece_fragment(m_piece.size()); - m_web.restart_request.piece = -1; + m_web->restart_request.piece = -1; } #if 0 @@ -576,7 +576,7 @@ void web_peer_connection::on_receive(error_code const& error { incoming_choke(); if (m_num_responses == 1) - m_web.supports_keepalive = false; + m_web->supports_keepalive = false; } #ifdef TORRENT_VERBOSE_LOGGING @@ -623,6 +623,7 @@ void web_peer_connection::on_receive(error_code const& error { // we should not try this server again. t->remove_web_seed(this, errors::missing_location, op_bittorrent, 2); + m_web = NULL; TORRENT_ASSERT(is_disconnecting()); #ifdef TORRENT_DEBUG TORRENT_ASSERT(statistics().last_payload_downloaded() @@ -660,6 +661,7 @@ void web_peer_connection::on_receive(error_code const& error if (i == std::string::npos) { t->remove_web_seed(this, errors::invalid_redirection, op_bittorrent, 2); + m_web = NULL; TORRENT_ASSERT(is_disconnecting()); #ifdef TORRENT_DEBUG TORRENT_ASSERT(statistics().last_payload_downloaded() @@ -670,11 +672,17 @@ void web_peer_connection::on_receive(error_code const& error } location.resize(i); } + else + { + location = resolve_redirect_location(m_url, location); + } + #ifdef TORRENT_VERBOSE_LOGGING peer_log("*** LOCATION: %s", location.c_str()); #endif t->add_web_seed(location, web_seed_entry::url_seed, m_external_auth, m_extra_headers); t->remove_web_seed(this, errors::redirecting, op_bittorrent, 2); + m_web = NULL; TORRENT_ASSERT(is_disconnecting()); #ifdef TORRENT_DEBUG TORRENT_ASSERT(statistics().last_payload_downloaded() @@ -722,6 +730,7 @@ void web_peer_connection::on_receive(error_code const& error received_bytes(0, bytes_transferred); // we should not try this server again. t->remove_web_seed(this, errors::invalid_range, op_bittorrent); + m_web = NULL; TORRENT_ASSERT(is_disconnecting()); #ifdef TORRENT_DEBUG TORRENT_ASSERT(statistics().last_payload_downloaded() @@ -742,6 +751,7 @@ void web_peer_connection::on_receive(error_code const& error received_bytes(0, bytes_transferred); // we should not try this server again. t->remove_web_seed(this, errors::no_content_length, op_bittorrent, 2); + m_web = NULL; TORRENT_ASSERT(is_disconnecting()); #ifdef TORRENT_DEBUG TORRENT_ASSERT(statistics().last_payload_downloaded() diff --git a/test/Jamfile b/test/Jamfile index 9c7ef3dcc..022de4f56 100644 --- a/test/Jamfile +++ b/test/Jamfile @@ -137,6 +137,7 @@ test-suite libtorrent : [ run test_checking.cpp ] [ run test_url_seed.cpp ] [ run test_web_seed.cpp ] + [ run test_web_seed_redirect.cpp ] [ run test_web_seed_socks4.cpp ] [ run test_web_seed_socks5.cpp ] [ run test_web_seed_socks5_pw.cpp ] diff --git a/test/test_http_parser.cpp b/test/test_http_parser.cpp index 404c074ca..ea1279256 100644 --- a/test/test_http_parser.cpp +++ b/test/test_http_parser.cpp @@ -351,6 +351,24 @@ int test_main() parse_url_components("http:", ec); TEST_CHECK(ec == error_code(errors::unsupported_url_protocol)); ec.clear(); + + // test resolve_redirect_location + + TEST_EQUAL(resolve_redirect_location("http://example.com/a/b", "a") + , "http://example.com/a/a"); + + TEST_EQUAL(resolve_redirect_location("http://example.com/a/b", "c/d/e/") + , "http://example.com/a/c/d/e/"); + + TEST_EQUAL(resolve_redirect_location("http://example.com/a/b", "../a") + , "http://example.com/a/../a"); + + TEST_EQUAL(resolve_redirect_location("http://example.com/a/b", "/c") + , "http://example.com/c"); + + TEST_EQUAL(resolve_redirect_location("http://example.com/a/b", "http://test.com/d") + , "http://test.com/d"); + return 0; } diff --git a/test/test_web_seed_redirect.cpp b/test/test_web_seed_redirect.cpp new file mode 100644 index 000000000..61a0b51fd --- /dev/null +++ b/test/test_web_seed_redirect.cpp @@ -0,0 +1,109 @@ +/* + +Copyright (c) 2008-2014, Arvid Norberg +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the distribution. + * Neither the name of the author nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. + +*/ + +#include "test.hpp" +#include "setup_transfer.hpp" +#include "web_seed_suite.hpp" +#include "libtorrent/create_torrent.hpp" + +using namespace libtorrent; + +const int proxy = libtorrent::proxy_settings::none; + +//static unsigned char random_byte() +//{ return std::rand() & 0xff; } + +int test_main() +{ + using namespace libtorrent; + + error_code ec; + + file_storage fs; + int piece_size = 0x4000; + + char random_data[16000]; + std::generate(random_data, random_data + sizeof(random_data), random_byte); + file f("test_file", file::write_only, ec); + if (ec) + { + fprintf(stderr, "failed to create file \"test_file\": (%d) %s\n" + , ec.value(), ec.message().c_str()); + return 1; + } + file::iovec_t b = { random_data, size_t(16000)}; + f.writev(0, &b, 1, ec); + fs.add_file("test_file", 16000); + + int port = start_web_server(); + + // generate a torrent with pad files to make sure they + // are not requested web seeds + libtorrent::create_torrent t(fs, piece_size, 0x4000); + + char tmp[512]; + snprintf(tmp, sizeof(tmp), "http://127.0.0.1:%d/redirect", port); + t.add_url_seed(tmp); + + // calculate the hash for all pieces + set_piece_hashes(t, ".", ec); + + if (ec) + { + fprintf(stderr, "error creating hashes for test torrent: %s\n" + , ec.message().c_str()); + TEST_CHECK(false); + return 0; + } + + std::vector buf; + bencode(std::back_inserter(buf), t.generate()); + boost::shared_ptr torrent_file(new torrent_info(&buf[0] + , buf.size(), ec)); + + { + session ses(fingerprint(" ", 0,0,0,0), 0); + session_settings settings; + settings.max_queued_disk_bytes = 256 * 1024; + ses.set_settings(settings); + ses.set_alert_mask(~(alert::progress_notification | alert::stats_notification)); + + // disable keep-alive because otherwise the test will choke on seeing + // the disconnect (from the redirect) + test_transfer(ses, torrent_file, 0, 0, "http", true, false, false, false); + } + + stop_web_server(); + return 0; +} + + diff --git a/test/web_seed_suite.cpp b/test/web_seed_suite.cpp index 69c640164..294e8be04 100644 --- a/test/web_seed_suite.cpp +++ b/test/web_seed_suite.cpp @@ -79,7 +79,7 @@ static sha1_hash file_hash(std::string const& name) static char const* proxy_name[] = {"", "_socks4", "_socks5", "_socks5_pw", "_http", "_http_pw", "_i2p"}; // proxy: 0=none, 1=socks4, 2=socks5, 3=socks5_pw 4=http 5=http_pw -static void test_transfer(lt::session& ses, boost::shared_ptr torrent_file +void test_transfer(lt::session& ses, boost::shared_ptr torrent_file , int proxy, int port, char const* protocol, bool url_seed , bool chunked_encoding, bool test_ban, bool keepalive) { diff --git a/test/web_seed_suite.hpp b/test/web_seed_suite.hpp index 1b71a12b2..50c0ca2a3 100644 --- a/test/web_seed_suite.hpp +++ b/test/web_seed_suite.hpp @@ -35,3 +35,9 @@ int EXPORT run_http_suite(int proxy, char const* protocol , bool test_url_seed, bool chunked_encoding = false, bool test_ban = false , bool keepalive = true, bool test_rename = false); +void EXPORT test_transfer(libtorrent::session& ses + , boost::shared_ptr torrent_file + , int proxy = 0, int port = 0, char const* protocol = "http" + , bool url_seed = true, bool chunked_encoding = false + , bool test_ban = false, bool keepalive = true); +