From 72322dbc103d904ea088337e81640f451bc7c37c Mon Sep 17 00:00:00 2001 From: Arvid Norberg Date: Sun, 10 Oct 2010 18:43:58 +0000 Subject: [PATCH] allow extending web seeds with extra headers and custom authorization schemese --- CMakeLists.txt | 1 + ChangeLog | 1 + Jamfile | 1 + docs/manual.rst | 58 ++++-- docs/tuning.rst | 15 ++ include/libtorrent/http_seed_connection.hpp | 51 +---- include/libtorrent/torrent.hpp | 59 ++---- include/libtorrent/torrent_info.hpp | 75 +++++++- include/libtorrent/web_connection_base.hpp | 168 ++++++++++++++++ include/libtorrent/web_peer_connection.hpp | 46 +---- src/Makefile.am | 1 + src/http_seed_connection.cpp | 113 +---------- src/session_impl.cpp | 3 +- src/torrent.cpp | 16 +- src/torrent_info.cpp | 61 +++++- src/web_connection_base.cpp | 200 ++++++++++++++++++++ src/web_peer_connection.cpp | 149 ++------------- 17 files changed, 607 insertions(+), 411 deletions(-) create mode 100644 include/libtorrent/web_connection_base.hpp create mode 100644 src/web_connection_base.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 890fadfab..427f3fac0 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -2,6 +2,7 @@ cmake_minimum_required(VERSION 2.6) project(libtorrent) set(sources + web_connection_base alert allocator assert diff --git a/ChangeLog b/ChangeLog index 3a00e7baf..155ee6d55 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,4 @@ + * support extending web seeds with custom authorization and extra headers * settings that are not changed from the default values are not saved in the session state * made seeding choking algorithm configurable diff --git a/Jamfile b/Jamfile index 602513fe8..dc0a558c4 100755 --- a/Jamfile +++ b/Jamfile @@ -375,6 +375,7 @@ SOURCES = ip_filter peer_connection bt_peer_connection + web_connection_base web_peer_connection http_seed_connection i2p_stream diff --git a/docs/manual.rst b/docs/manual.rst index c2178b8e9..01658d747 100644 --- a/docs/manual.rst +++ b/docs/manual.rst @@ -1385,10 +1385,9 @@ The ``torrent_info`` has the following synopsis:: bool priv() const; - std::vector const& url_seeds() const; void add_url_seed(std::string const& url); - std::vector const& http_seeds() const; void add_http_seed(std::string const& url); + std::vector const& web_seeds() const; size_type total_size() const; int piece_length() const; @@ -1646,24 +1645,59 @@ The input range is assumed to be valid within the torrent. ``file_offset`` must refer to a valid file, i.e. it cannot be >= ``num_files()``. -url_seeds() add_url_seed() http_seeds() add_http_seed() -------------------------------------------------------- +add_url_seed() add_http_seed() +------------------------------ :: - std::vector const& url_seeds() const; - void add_url_seed(std::string const& url); - std::vector const& http_seeds() const; - void add_http_seed(std::string const& url); + void add_url_seed(std::string const& url + , std::string const& extern_auth = std::string() + , web_seed_entry::headers_t const& extra_headers = web_seed_entry::headers_t()); + void add_http_seed(std::string const& url + , std::string const& extern_auth = std::string() + , web_seed_entry::headers_t const& extra_headers = web_seed_entry::headers_t()); + std::vector const& web_seeds() const; -If there are any url-seeds or http seeds in this torrent, ``url_seeds()`` -and ``http_seeds()`` will return a vector of those urls. +``web_seeds()`` returns all url seeds and http seeds in the torrent. Each entry +is a ``web_seed_entry`` and may refer to either a url seed or http seed. + ``add_url_seed()`` and ``add_http_seed()`` adds one url to the list of -url/http seeds. Currently, the only transport protocol -supported for the url is http. +url/http seeds. Currently, the only transport protocol supported for the url +is http. + +The ``extern_auth`` argument can be used for other athorization schemese than +basic HTTP authorization. If set, it will override any username and password +found in the URL itself. The string will be sent as the HTTP authorization header's +value (without specifying "Basic"). + +The ``extra_headers`` argument defaults to an empty list, but can be used to +insert custom HTTP headers in the requests to a specific web seed. See `HTTP seeding`_ for more information. +The ``web_seed_entry`` has the following members:: + + struct web_seed_entry + { + enum type_t { url_seed, http_seed }; + + typedef std::vector > headers_t; + + web_seed_entry(std::string const& url_, type_t type_ + , std::string const& auth_ = std::string() + , headers_t const& extra_headers_ = headers_t()); + + bool operator==(web_seed_entry const& e) const; + bool operator<(web_seed_entry const& e) const; + + std::string url; + type_t type; + std::string auth; + headers_t extra_headers; + + // ... + }; + trackers() ---------- diff --git a/docs/tuning.rst b/docs/tuning.rst index 749d6f207..14dd00d78 100644 --- a/docs/tuning.rst +++ b/docs/tuning.rst @@ -168,6 +168,21 @@ support, you need to patch parts of boost. Also make sure to optimize for size when compiling. +Another way of reducing the executable size is to disable code that isn't used. +There are a number of ``TORRENT_*`` macros that control which features are included +in libtorrent. If these macros are used to strip down libtorrent, make sure the same +macros are defined when building libtorrent as when linking against it. If these +are different the structures will look different from the libtorrent side and from +the client side and memory corruption will follow. + +One, probably, safe macro to define is ``TORRENT_NO_DEPRECATE`` which removes all +deprecated functions and struct members. As long as no deprecated functions are +relied upon, this should be a simple way to eliminate a little bit of code. + +For all available options, see the `building libtorrent`_ secion. + +.. _`building libtorrent`: building.html + reduce statistics ----------------- diff --git a/include/libtorrent/http_seed_connection.hpp b/include/libtorrent/http_seed_connection.hpp index 17f51daa5..8d34ad22f 100644 --- a/include/libtorrent/http_seed_connection.hpp +++ b/include/libtorrent/http_seed_connection.hpp @@ -56,7 +56,7 @@ POSSIBILITY OF SUCH DAMAGE. #endif #include "libtorrent/config.hpp" -#include "libtorrent/peer_connection.hpp" +#include "libtorrent/web_connection_base.hpp" #include "libtorrent/disk_buffer_holder.hpp" #include "libtorrent/torrent.hpp" #include "libtorrent/piece_block_progress.hpp" @@ -73,7 +73,7 @@ namespace libtorrent } class TORRENT_EXPORT http_seed_connection - : public peer_connection + : public web_connection_base { friend class invariant_access; public: @@ -87,43 +87,23 @@ namespace libtorrent , boost::shared_ptr s , tcp::endpoint const& remote , std::string const& url - , policy::peer* peerinfo); - void start(); + , policy::peer* peerinfo + , std::string const& ext_auth + , web_seed_entry::headers_t const& ext_headers); virtual int type() const { return peer_connection::http_seed_connection; } // called from the main loop when this connection has any // work to do. - void on_sent(error_code const& error - , std::size_t bytes_transferred); void on_receive(error_code const& error , std::size_t bytes_transferred); std::string const& url() const { return m_url; } virtual void get_specific_peer_info(peer_info& p) const; - virtual bool in_handshake() const; virtual void disconnect(error_code const& ec, int error = 0); - // the following functions appends messages - // to the send buffer - void write_choke() {} - void write_unchoke() {} - void write_interested() {} - void write_not_interested() {} void write_request(peer_request const& r); - void write_cancel(peer_request const& r) {} - void write_have(int index) {} - void write_piece(peer_request const& r, disk_buffer_holder& buffer) { TORRENT_ASSERT(false); } - void write_keepalive() {} - void on_connected(); - void write_reject_request(peer_request const&) {} - void write_allow_fast(int) {} - void write_suggest(int piece) {} - -#ifdef TORRENT_DEBUG - void check_invariant() const; -#endif private: @@ -134,32 +114,13 @@ namespace libtorrent // will be invalid. boost::optional downloading_piece_progress() const; - // this has one entry per bittorrent request - std::deque m_requests; - - std::string m_server_string; - http_parser m_parser; - std::string m_auth; - std::string m_host; - int m_port; - std::string m_path; - // this is const since it's used as a key in the web seed list in the torrent // if it's changed referencing back into that list will fail const std::string m_url; - // the first request will contain a little bit more data - // than subsequent ones, things that aren't critical are left - // out to save bandwidth. - bool m_first_request; - // the number of bytes left to receive of the response we're // currently parsing - int m_response_left; - - // the number of bytes in the start of the receive buffer - // that's http header - int m_body_start; + int m_response_left; }; } diff --git a/include/libtorrent/torrent.hpp b/include/libtorrent/torrent.hpp index 812a81d41..41bff841f 100644 --- a/include/libtorrent/torrent.hpp +++ b/include/libtorrent/torrent.hpp @@ -98,39 +98,6 @@ namespace libtorrent struct piece_checker_data; } - struct web_seed_entry - { - std::string url; - // http seeds are different from url seeds in the - // protocol they use. http seeds follows the original - // http seed spec. by John Hoffman - enum type_t { url_seed, http_seed } type; - - // if this is > now, we can't reconnect yet - ptime retry; - - // this indicates whether or not we're resolving the - // hostname of this URL - bool resolving; - - tcp::endpoint endpoint; - - peer_connection* connection; - - web_seed_entry(std::string const& url_, type_t type_) - : url(url_), type(type_), retry(time_now()), resolving(false), connection(0) {} - - bool operator==(web_seed_entry const& e) const - { return url == e.url && type == e.type; } - - bool operator<(web_seed_entry const& e) const - { - if (url < e.url) return true; - if (url > e.url) return false; - return type < e.type; - } - }; - // a torrent is a class that holds information // for a specific download. It updates itself against // the tracker @@ -365,8 +332,24 @@ namespace libtorrent // add or remove a url that will be attempted for // finding the file(s) in this torrent. void add_web_seed(std::string const& url, web_seed_entry::type_t type) - { m_web_seeds.push_back(web_seed_entry(url, type)); } + { + m_web_seeds.push_back(web_seed_entry(url, type)); + } + + void add_web_seed(std::string const& url, web_seed_entry::type_t type + , std::string const& auth, web_seed_entry::headers_t const& extra_headers) + { + m_web_seeds.push_back(web_seed_entry(url, type, auth, extra_headers)); + } + void remove_web_seed(std::string const& url, web_seed_entry::type_t type) + { + std::list::iterator i = std::find_if(m_web_seeds.begin(), m_web_seeds.end() + , (boost::bind(&web_seed_entry::url, _1) + == url && boost::bind(&web_seed_entry::type, _1) == type)); + if (i != m_web_seeds.end()) m_web_seeds.erase(i); + } + void disconnect_web_seed(peer_connection* p) { std::list::iterator i = std::find_if(m_web_seeds.begin(), m_web_seeds.end() @@ -379,14 +362,6 @@ namespace libtorrent i->connection = 0; } - void remove_web_seed(std::string const& url, web_seed_entry::type_t type) - { - std::list::iterator i = std::find_if(m_web_seeds.begin(), m_web_seeds.end() - , (boost::bind(&web_seed_entry::url, _1) - == url && boost::bind(&web_seed_entry::type, _1) == type)); - if (i != m_web_seeds.end()) m_web_seeds.erase(i); - } - void retry_web_seed(peer_connection* p, int retry = 0); void remove_web_seed(peer_connection* p) diff --git a/include/libtorrent/torrent_info.hpp b/include/libtorrent/torrent_info.hpp index f385924f1..fee913e2a 100644 --- a/include/libtorrent/torrent_info.hpp +++ b/include/libtorrent/torrent_info.hpp @@ -57,9 +57,12 @@ POSSIBILITY OF SUCH DAMAGE. #include "libtorrent/assert.hpp" #include "libtorrent/file_storage.hpp" #include "libtorrent/copy_ptr.hpp" +#include "libtorrent/socket.hpp" namespace libtorrent { + struct peer_connection; + enum { // wait 60 seconds before retrying a failed tracker @@ -168,6 +171,50 @@ namespace libtorrent void trim(); }; + struct web_seed_entry + { + // http seeds are different from url seeds in the + // protocol they use. http seeds follows the original + // http seed spec. by John Hoffman + enum type_t { url_seed, http_seed }; + + typedef std::vector > headers_t; + + web_seed_entry(std::string const& url_, type_t type_ + , std::string const& auth_ = std::string() + , headers_t const& extra_headers_ = headers_t()) + : url(url_), type(type_) + , auth(auth_), extra_headers(extra_headers_) + , retry(time_now()), resolving(false), connection(0) + {} + + bool operator==(web_seed_entry const& e) const + { return url == e.url && type == e.type; } + + bool operator<(web_seed_entry const& e) const + { + if (url < e.url) return true; + if (url > e.url) return false; + return type < e.type; + } + + std::string url; + type_t type; + std::string auth; + headers_t extra_headers; + + // if this is > now, we can't reconnect yet + ptime retry; + + // this indicates whether or not we're resolving the + // hostname of this URL + bool resolving; + + tcp::endpoint endpoint; + + peer_connection* connection; + }; + #ifndef BOOST_NO_EXCEPTIONS // for backwards compatibility with 0.14 typedef libtorrent_exception invalid_torrent_file; @@ -220,15 +267,24 @@ namespace libtorrent void add_tracker(std::string const& url, int tier = 0); std::vector const& trackers() const { return m_urls; } - std::vector const& url_seeds() const - { return m_url_seeds; } - void add_url_seed(std::string const& url) - { m_url_seeds.push_back(url); } +#ifndef TORRENT_NO_DEPRECATE + // deprecated in 0.16. Use web_seeds() instead + TORRENT_DEPRECATED_PREFIX + std::vector url_seeds() const TORRENT_DEPRECATED; + TORRENT_DEPRECATED_PREFIX + std::vector http_seeds() const TORRENT_DEPRECATED; +#endif // TORRENT_NO_DEPRECATE - std::vector const& http_seeds() const - { return m_http_seeds; } - void add_http_seed(std::string const& url) - { m_http_seeds.push_back(url); } + void add_url_seed(std::string const& url + , std::string const& extern_auth = std::string() + , web_seed_entry::headers_t const& extra_headers = web_seed_entry::headers_t()); + + void add_http_seed(std::string const& url + , std::string const& extern_auth = std::string() + , web_seed_entry::headers_t const& extra_headers = web_seed_entry::headers_t()); + + std::vector const& web_seeds() const + { return m_web_seeds; } size_type total_size() const { return m_files.total_size(); } int piece_length() const { return m_files.piece_length(); } @@ -358,8 +414,7 @@ namespace libtorrent // the urls to the trackers std::vector m_urls; - std::vector m_url_seeds; - std::vector m_http_seeds; + std::vector m_web_seeds; nodes_t m_nodes; // if this is a merkle torrent, this is the merkle diff --git a/include/libtorrent/web_connection_base.hpp b/include/libtorrent/web_connection_base.hpp new file mode 100644 index 000000000..0722f0383 --- /dev/null +++ b/include/libtorrent/web_connection_base.hpp @@ -0,0 +1,168 @@ +/* + +Copyright (c) 2003, Arvid Norberg +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the distribution. + * Neither the name of the author nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. + +*/ + +#ifndef WEB_CONNECTION_BASE_HPP_INCLUDED +#define WEB_CONNECTION_BASE_HPP_INCLUDED + +#include +#include +#include +#include +#include + +#include "libtorrent/debug.hpp" + +#ifdef _MSC_VER +#pragma warning(push, 1) +#endif + +#include +#include +#include +#include +#include +#include + +#ifdef _MSC_VER +#pragma warning(pop) +#endif + +#include "libtorrent/buffer.hpp" +#include "libtorrent/peer_connection.hpp" +#include "libtorrent/socket.hpp" +#include "libtorrent/peer_id.hpp" +#include "libtorrent/storage.hpp" +#include "libtorrent/stat.hpp" +#include "libtorrent/alert.hpp" +#include "libtorrent/torrent_handle.hpp" +#include "libtorrent/torrent.hpp" +#include "libtorrent/peer_request.hpp" +#include "libtorrent/piece_block_progress.hpp" +#include "libtorrent/config.hpp" +// parse_url +#include "libtorrent/tracker_manager.hpp" +#include "libtorrent/http_parser.hpp" + +namespace libtorrent +{ + class torrent; + + namespace detail + { + struct session_impl; + } + + class TORRENT_EXPORT web_connection_base + : public peer_connection + { + friend class invariant_access; + public: + + // this is the constructor where the we are the active part. + // The peer_conenction should handshake and verify that the + // other end has the correct id + web_connection_base( + aux::session_impl& ses + , boost::weak_ptr t + , boost::shared_ptr s + , tcp::endpoint const& remote + , std::string const& url + , policy::peer* peerinfo + , std::string const& ext_auth + , web_seed_entry::headers_t const& ext_headers); + void start(); + + ~web_connection_base(); + + // called from the main loop when this connection has any + // work to do. + void on_sent(error_code const& error + , std::size_t bytes_transferred); + + virtual std::string const& url() const = 0; + + bool in_handshake() const; + + // the following functions appends messages + // to the send buffer + void write_choke() {} + void write_unchoke() {} + void write_interested() {} + void write_not_interested() {} + virtual void write_request(peer_request const& r) = 0; + void write_cancel(peer_request const& r) + { incoming_reject_request(r); } + void write_have(int index) {} + void write_piece(peer_request const& r, disk_buffer_holder& buffer) { TORRENT_ASSERT(false); } + void write_keepalive() {} + void on_connected(); + void write_reject_request(peer_request const&) {} + void write_allow_fast(int) {} + void write_suggest(int piece) {} + +#ifdef TORRENT_DEBUG + void check_invariant() const; +#endif + + virtual void get_specific_peer_info(peer_info& p) const; + + protected: + + virtual void add_headers(std::string& request + , proxy_settings const& ps, bool using_proxy) const; + + // this has one entry per bittorrent request + std::deque m_requests; + + std::string m_server_string; + http_parser m_parser; + std::string m_basic_auth; + std::string m_host; + int m_port; + std::string m_path; + + std::string m_external_auth; + web_seed_entry::headers_t m_extra_headers; + + // the first request will contain a little bit more data + // than subsequent ones, things that aren't critical are left + // out to save bandwidth. + bool m_first_request; + + // the number of bytes into the receive buffer where + // current read cursor is. + int m_body_start; + }; +} + +#endif // TORRENT_WEB_CONNECTION_BASE_HPP_INCLUDED + diff --git a/include/libtorrent/web_peer_connection.hpp b/include/libtorrent/web_peer_connection.hpp index 0c0860137..162003b66 100644 --- a/include/libtorrent/web_peer_connection.hpp +++ b/include/libtorrent/web_peer_connection.hpp @@ -55,7 +55,7 @@ POSSIBILITY OF SUCH DAMAGE. #endif #include "libtorrent/config.hpp" -#include "libtorrent/peer_connection.hpp" +#include "libtorrent/web_connection_base.hpp" #include "libtorrent/disk_buffer_holder.hpp" #include "libtorrent/torrent.hpp" #include "libtorrent/piece_block_progress.hpp" @@ -71,7 +71,7 @@ namespace libtorrent } class TORRENT_EXPORT web_peer_connection - : public peer_connection + : public web_connection_base { friend class invariant_access; public: @@ -85,43 +85,23 @@ namespace libtorrent , boost::shared_ptr s , tcp::endpoint const& remote , std::string const& url - , policy::peer* peerinfo); - void start(); + , policy::peer* peerinfo + , std::string const& ext_auth + , web_seed_entry::headers_t const& ext_headers); virtual int type() const { return peer_connection::url_seed_connection; } // called from the main loop when this connection has any // work to do. - void on_sent(error_code const& error - , std::size_t bytes_transferred); void on_receive(error_code const& error , std::size_t bytes_transferred); std::string const& url() const { return m_original_url; } virtual void get_specific_peer_info(peer_info& p) const; - virtual bool in_handshake() const; virtual void disconnect(error_code const& ec, int error = 0); - // the following functions appends messages - // to the send buffer - void write_choke() {} - void write_unchoke() {} - void write_interested() {} - void write_not_interested() {} void write_request(peer_request const& r); - void write_cancel(peer_request const& r) {} - void write_have(int index) {} - void write_piece(peer_request const& r, disk_buffer_holder& buffer) { TORRENT_ASSERT(false); } - void write_keepalive() {} - void on_connected(); - void write_reject_request(peer_request const&) {} - void write_allow_fast(int) {} - void write_suggest(int piece) {} - -#ifdef TORRENT_DEBUG - void check_invariant() const; -#endif private: @@ -132,33 +112,17 @@ namespace libtorrent // will be invalid. boost::optional downloading_piece_progress() const; - // this has one entry per bittorrent request - std::deque m_requests; // this has one entry per http-request // (might be more than the bt requests) std::deque m_file_requests; - std::string m_server_string; - http_parser m_parser; - std::string m_auth; - std::string m_host; - int m_port; - std::string m_path; std::string m_url; std::string m_original_url; - // the first request will contain a little bit more data - // than subsequent ones, things that aren't critical are left - // out to save bandwidth. - bool m_first_request; - // this is used for intermediate storage of pieces // that are received in more than one HTTP response std::vector m_piece; - // the number of bytes into the receive buffer where - // current read cursor is. - int m_body_start; // the number of bytes received in the current HTTP // response. used to know where in the buffer the // next response starts diff --git a/src/Makefile.am b/src/Makefile.am index ed16a6b83..84e75813e 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -17,6 +17,7 @@ GEOIP_SOURCES = GeoIP.c endif libtorrent_rasterbar_la_SOURCES = \ + web_connection_base.cpp \ alert.cpp \ allocator.cpp \ assert.cpp \ diff --git a/src/http_seed_connection.cpp b/src/http_seed_connection.cpp index 864a440f6..84aba9aab 100644 --- a/src/http_seed_connection.cpp +++ b/src/http_seed_connection.cpp @@ -60,25 +60,17 @@ namespace libtorrent , boost::shared_ptr s , tcp::endpoint const& remote , std::string const& url - , policy::peer* peerinfo) - : peer_connection(ses, t, s, remote, peerinfo) + , policy::peer* peerinfo + , std::string const& auth + , web_seed_entry::headers_t const& extra_headers) + : web_connection_base(ses, t, s, remote, url, peerinfo, auth, extra_headers) , m_url(url) - , m_first_request(true) - , m_response_left(0) - , m_body_start(0) { INVARIANT_CHECK; if (!ses.settings().report_web_seed_downloads) ignore_stats(true); - // we want large blocks as well, so - // we can request more bytes at once - request_large_blocks(true); - prefer_whole_pieces(1); - - // we only want left-over bandwidth - set_priority(1); shared_ptr tor = t.lock(); TORRENT_ASSERT(tor); int blocks_per_piece = tor->torrent_file().piece_length() / tor->block_size(); @@ -88,31 +80,11 @@ namespace libtorrent m_max_out_request_queue = ses.settings().urlseed_pipeline_size * blocks_per_piece; - // since this is a web seed, change the timeout - // according to the settings. - set_timeout(ses.settings().urlseed_timeout); + prefer_whole_pieces(1); + #ifdef TORRENT_VERBOSE_LOGGING (*m_logger) << "*** http_seed_connection\n"; #endif - - std::string protocol; - error_code ec; - boost::tie(protocol, m_auth, m_host, m_port, m_path) - = parse_url_components(url, ec); - TORRENT_ASSERT(!ec); - - if (!m_auth.empty()) - m_auth = base64encode(m_auth); - - m_server_string = "HTTP seed @ "; - m_server_string += m_host; - } - - void http_seed_connection::start() - { - set_upload_only(true); - if (is_disconnecting()) return; - peer_connection::start(); } void http_seed_connection::disconnect(error_code const& ec, int error) @@ -154,20 +126,6 @@ namespace libtorrent return ret; } - void http_seed_connection::on_connected() - { - boost::shared_ptr t = associated_torrent().lock(); - TORRENT_ASSERT(t); - - // this is always a seed - incoming_have_all(); - - // it is always possible to request pieces - incoming_unchoke(); - - reset_recv_buffer(t->block_size() + 1024); - } - void http_seed_connection::write_request(peer_request const& r) { INVARIANT_CHECK; @@ -220,29 +178,7 @@ namespace libtorrent } request += " HTTP/1.1\r\n"; - request += "Host: "; - request += m_host; - if (m_first_request && !m_ses.settings().user_agent.empty()) - { - request += "\r\nUser-Agent: "; - request += m_ses.settings().user_agent; - } - if (!m_auth.empty()) - { - request += "\r\nAuthorization: Basic "; - request += m_auth; - } - if (ps.type == proxy_settings::http_pw) - { - request += "\r\nProxy-Authorization: Basic "; - request += base64encode(ps.username + ":" + ps.password); - } - if (using_proxy) - { - request += "\r\nProxy-Connection: keep-alive"; - } - if (m_first_request || using_proxy) - request += "\r\nConnection: keep-alive"; + add_headers(request, ps, using_proxy); request += "\r\n\r\n"; m_first_request = false; @@ -440,43 +376,10 @@ namespace libtorrent void http_seed_connection::get_specific_peer_info(peer_info& p) const { - if (is_interesting()) p.flags |= peer_info::interesting; - if (is_choked()) p.flags |= peer_info::choked; + web_connection_base::get_specific_peer_info(p); p.flags |= peer_info::local_connection; - if (!is_connecting() && m_server_string.empty()) - p.flags |= peer_info::handshake; - if (is_connecting() && !is_queued()) p.flags |= peer_info::connecting; - if (is_queued()) p.flags |= peer_info::queued; - - p.client = m_server_string; p.connection_type = peer_info::http_seed; } - bool http_seed_connection::in_handshake() const - { - return m_server_string.empty(); - } - - void http_seed_connection::on_sent(error_code const& error - , std::size_t bytes_transferred) - { - INVARIANT_CHECK; - - if (error) return; - m_statistics.sent_bytes(0, bytes_transferred); - } - - -#ifdef TORRENT_DEBUG - void http_seed_connection::check_invariant() const - { -/* - TORRENT_ASSERT(m_num_pieces == std::count( - m_have_piece.begin() - , m_have_piece.end() - , true)); -*/ } -#endif - } diff --git a/src/session_impl.cpp b/src/session_impl.cpp index 432ab959e..438510747 100644 --- a/src/session_impl.cpp +++ b/src/session_impl.cpp @@ -593,8 +593,7 @@ namespace aux { PRINT_SIZEOF(torrent_info) PRINT_OFFSETOF(torrent_info, m_files) PRINT_OFFSETOF(torrent_info, m_orig_files) - PRINT_OFFSETOF(torrent_info, m_url_seeds) - PRINT_OFFSETOF(torrent_info, m_http_seeds) + PRINT_OFFSETOF(torrent_info, m_web_seeds) PRINT_OFFSETOF(torrent_info, m_nodes) PRINT_OFFSETOF(torrent_info, m_merkle_tree) PRINT_OFFSETOF(torrent_info, m_info_section) diff --git a/src/torrent.cpp b/src/torrent.cpp index 603ebad2b..2923950d1 100644 --- a/src/torrent.cpp +++ b/src/torrent.cpp @@ -893,15 +893,9 @@ namespace libtorrent // ans also in the case of share mode, we need to update the priorities update_piece_priorities(); - std::vector const& url_seeds = m_torrent_file->url_seeds(); - for (std::vector::const_iterator i = url_seeds.begin() - , end(url_seeds.end()); i != end; ++i) - add_web_seed(*i, web_seed_entry::url_seed); - std::vector const& http_seeds = m_torrent_file->http_seeds(); - for (std::vector::const_iterator i = http_seeds.begin() - , end(http_seeds.end()); i != end; ++i) - add_web_seed(*i, web_seed_entry::http_seed); + std::vector const& web_seeds = m_torrent_file->web_seeds(); + m_web_seeds.insert(m_web_seeds.end(), web_seeds.begin(), web_seeds.end()); if (m_seed_mode) { @@ -3571,12 +3565,14 @@ namespace libtorrent if (web->type == web_seed_entry::url_seed) { c = new (std::nothrow) web_peer_connection( - m_ses, shared_from_this(), s, a, web->url, 0); + m_ses, shared_from_this(), s, a, web->url, 0, // TODO: pass in web + web->auth, web->extra_headers); } else if (web->type == web_seed_entry::http_seed) { c = new (std::nothrow) http_seed_connection( - m_ses, shared_from_this(), s, a, web->url, 0); + m_ses, shared_from_this(), s, a, web->url, 0, // TODO: pass in web + web->auth, web->extra_headers); } if (!c) return; diff --git a/src/torrent_info.cpp b/src/torrent_info.cpp index f103f8070..a11c8033c 100644 --- a/src/torrent_info.cpp +++ b/src/torrent_info.cpp @@ -440,8 +440,7 @@ namespace libtorrent : m_files(t.m_files) , m_orig_files(t.m_orig_files) , m_urls(t.m_urls) - , m_url_seeds(t.m_url_seeds) - , m_http_seeds(t.m_http_seeds) + , m_web_seeds(t.m_web_seeds) , m_nodes(t.m_nodes) , m_merkle_tree(t.m_merkle_tree) , m_piece_hashes(t.m_piece_hashes) @@ -703,7 +702,7 @@ namespace libtorrent { using std::swap; m_urls.swap(ti.m_urls); - m_url_seeds.swap(ti.m_url_seeds); + m_web_seeds.swap(ti.m_web_seeds); m_files.swap(ti.m_files); m_orig_files.swap(ti.m_orig_files); m_nodes.swap(ti.m_nodes); @@ -1065,7 +1064,8 @@ namespace libtorrent lazy_entry const* url_seeds = torrent_file.dict_find("url-list"); if (url_seeds && url_seeds->type() == lazy_entry::string_t) { - m_url_seeds.push_back(maybe_url_encode(url_seeds->string_value())); + m_web_seeds.push_back(web_seed_entry(maybe_url_encode(url_seeds->string_value()) + , web_seed_entry::url_seed)); } else if (url_seeds && url_seeds->type() == lazy_entry::list_t) { @@ -1073,7 +1073,8 @@ namespace libtorrent { lazy_entry const* url = url_seeds->list_at(i); if (url->type() != lazy_entry::string_t) continue; - m_url_seeds.push_back(maybe_url_encode(url->string_value())); + m_web_seeds.push_back(web_seed_entry(maybe_url_encode(url->string_value()) + , web_seed_entry::url_seed)); } } @@ -1081,7 +1082,8 @@ namespace libtorrent lazy_entry const* http_seeds = torrent_file.dict_find("httpseeds"); if (http_seeds && http_seeds->type() == lazy_entry::string_t) { - m_http_seeds.push_back(maybe_url_encode(http_seeds->string_value())); + m_web_seeds.push_back(web_seed_entry(maybe_url_encode(http_seeds->string_value()) + , web_seed_entry::http_seed)); } else if (http_seeds && http_seeds->type() == lazy_entry::list_t) { @@ -1089,7 +1091,8 @@ namespace libtorrent { lazy_entry const* url = http_seeds->list_at(i); if (url->type() != lazy_entry::string_t) continue; - m_http_seeds.push_back(maybe_url_encode(url->string_value())); + m_web_seeds.push_back(web_seed_entry(maybe_url_encode(url->string_value()) + , web_seed_entry::http_seed)); } } @@ -1131,6 +1134,50 @@ namespace libtorrent < boost::bind(&announce_entry::tier, _2)); } +#ifndef TORRENT_NO_DEPRECATE + namespace + { + struct filter_web_seed_type + { + filter_web_seed_type(web_seed_entry::type_t t_) : t(t_) {} + void operator() (web_seed_entry const& w) + { if (w.type == t) urls.push_back(w.url); } + std::vector urls; + web_seed_entry::type_t t; + }; + } + + std::vector torrent_info::url_seeds() const + { + return std::for_each(m_web_seeds.begin(), m_web_seeds.end() + , filter_web_seed_type(web_seed_entry::url_seed)).urls; + } + + std::vector torrent_info::http_seeds() const + { + return std::for_each(m_web_seeds.begin(), m_web_seeds.end() + , filter_web_seed_type(web_seed_entry::http_seed)).urls; + } + +#endif // TORRENT_NO_DEPRECATE + + void torrent_info::add_url_seed(std::string const& url + , std::string const& ext_auth + , web_seed_entry::headers_t const& ext_headers) + { + m_web_seeds.push_back(web_seed_entry(url, web_seed_entry::url_seed + , ext_auth, ext_headers)); + } + + void torrent_info::add_http_seed(std::string const& url + , std::string const& auth + , web_seed_entry::headers_t const& extra_headers) + { + m_web_seeds.push_back(web_seed_entry(url, web_seed_entry::http_seed + , auth, extra_headers)); + } + + #if !defined TORRENT_NO_DEPRECATE && TORRENT_USE_IOSTREAM // ------- start deprecation ------- diff --git a/src/web_connection_base.cpp b/src/web_connection_base.cpp new file mode 100644 index 000000000..c740b0fcc --- /dev/null +++ b/src/web_connection_base.cpp @@ -0,0 +1,200 @@ +/* + +Copyright (c) 2003, Arvid Norberg +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the distribution. + * Neither the name of the author nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. + +*/ + +#include "libtorrent/pch.hpp" + +#include +#include +#include +#include + +#include "libtorrent/web_connection_base.hpp" +#include "libtorrent/session.hpp" +#include "libtorrent/identify_client.hpp" +#include "libtorrent/entry.hpp" +#include "libtorrent/bencode.hpp" +#include "libtorrent/alert_types.hpp" +#include "libtorrent/invariant_check.hpp" +#include "libtorrent/io.hpp" +#include "libtorrent/version.hpp" +#include "libtorrent/aux_/session_impl.hpp" +#include "libtorrent/parse_url.hpp" +#include "libtorrent/peer_info.hpp" + +using boost::shared_ptr; +using libtorrent::aux::session_impl; + +namespace libtorrent +{ + web_connection_base::web_connection_base( + session_impl& ses + , boost::weak_ptr t + , boost::shared_ptr s + , tcp::endpoint const& remote + , std::string const& url + , policy::peer* peerinfo + , std::string const& auth + , web_seed_entry::headers_t const& extra_headers) + : peer_connection(ses, t, s, remote, peerinfo) + , m_first_request(true) + , m_external_auth(auth) + , m_extra_headers(extra_headers) + { + INVARIANT_CHECK; + + // we want large blocks as well, so + // we can request more bytes at once + request_large_blocks(true); + + // we only want left-over bandwidth + set_priority(1); + + // since this is a web seed, change the timeout + // according to the settings. + set_timeout(ses.settings().urlseed_timeout); + + std::string protocol; + error_code ec; + boost::tie(protocol, m_basic_auth, m_host, m_port, m_path) + = parse_url_components(url, ec); + TORRENT_ASSERT(!ec); + + if (!m_basic_auth.empty()) + m_basic_auth = base64encode(m_basic_auth); + + m_server_string = "URL seed @ "; + m_server_string += m_host; + } + + void web_connection_base::start() + { + set_upload_only(true); + if (is_disconnecting()) return; + peer_connection::start(); + } + + web_connection_base::~web_connection_base() + {} + + void web_connection_base::on_connected() + { + boost::shared_ptr t = associated_torrent().lock(); + TORRENT_ASSERT(t); + + // this is always a seed + incoming_have_all(); + + // it is always possible to request pieces + incoming_unchoke(); + + reset_recv_buffer(t->block_size() + 1024); + } + + void web_connection_base::add_headers(std::string& request + , proxy_settings const& ps, bool using_proxy) const + { + request += "Host: "; + request += m_host; + if (m_first_request) { + request += "\r\nUser-Agent: "; + request += m_ses.settings().user_agent; + } + if (!m_external_auth.empty()) { + request += "\r\nAuthorization: "; + request += m_external_auth; + } else if (!m_basic_auth.empty()) { + request += "\r\nAuthorization: Basic "; + request += m_basic_auth; + } + if (ps.type == proxy_settings::http_pw) { + request += "\r\nProxy-Authorization: Basic "; + request += base64encode(ps.username + ":" + ps.password); + } + for (web_seed_entry::headers_t::const_iterator it = m_extra_headers.begin(); + it != m_extra_headers.end(); ++it) { + request += "\r\n"; + request += it->first; + request += ": "; + request += it->second; + } + if (using_proxy) { + request += "\r\nProxy-Connection: keep-alive"; + } + if (m_first_request || using_proxy) { + request += "\r\nConnection: keep-alive"; + } + } + + // -------------------------- + // RECEIVE DATA + // -------------------------- + + void web_connection_base::get_specific_peer_info(peer_info& p) const + { + if (is_interesting()) p.flags |= peer_info::interesting; + if (is_choked()) p.flags |= peer_info::choked; + if (!is_connecting() && m_server_string.empty()) + p.flags |= peer_info::handshake; + if (is_connecting() && !is_queued()) p.flags |= peer_info::connecting; + if (is_queued()) p.flags |= peer_info::queued; + + p.client = m_server_string; + } + + bool web_connection_base::in_handshake() const + { + return m_server_string.empty(); + } + + void web_connection_base::on_sent(error_code const& error + , std::size_t bytes_transferred) + { + INVARIANT_CHECK; + + if (error) return; + m_statistics.sent_bytes(0, bytes_transferred); + } + + +#ifdef TORRENT_DEBUG + void web_connection_base::check_invariant() const + { +/* + TORRENT_ASSERT(m_num_pieces == std::count( + m_have_piece.begin() + , m_have_piece.end() + , true)); +*/ } +#endif + +} + diff --git a/src/web_peer_connection.cpp b/src/web_peer_connection.cpp index f555a9134..0b023deb0 100644 --- a/src/web_peer_connection.cpp +++ b/src/web_peer_connection.cpp @@ -61,10 +61,11 @@ namespace libtorrent , boost::shared_ptr s , tcp::endpoint const& remote , std::string const& url - , policy::peer* peerinfo) - : peer_connection(ses, t, s, remote, peerinfo) + , policy::peer* peerinfo + , std::string const& auth + , web_seed_entry::headers_t const& extra_headers) + : web_connection_base(ses, t, s, remote, url, peerinfo, auth, extra_headers) , m_url(url) - , m_first_request(true) , m_range_pos(0) , m_block_pos(0) { @@ -73,12 +74,6 @@ namespace libtorrent if (!ses.settings().report_web_seed_downloads) ignore_stats(true); - // we want large blocks as well, so - // we can request more bytes at once - request_large_blocks(true); - - // we only want left-over bandwidth - set_priority(1); shared_ptr tor = t.lock(); TORRENT_ASSERT(tor); int blocks_per_piece = tor->torrent_file().piece_length() / tor->block_size(); @@ -87,36 +82,9 @@ namespace libtorrent // from web seeds prefer_whole_pieces((1024 * 1024) / tor->torrent_file().piece_length()); - // multiply with the blocks per piece since that many requests are - // merged into one http request - m_max_out_request_queue = ses.settings().urlseed_pipeline_size - * blocks_per_piece; - - // since this is a web seed, change the timeout - // according to the settings. - set_timeout(ses.settings().urlseed_timeout); #ifdef TORRENT_VERBOSE_LOGGING (*m_logger) << "*** web_peer_connection " << url << "\n"; #endif - - std::string protocol; - error_code ec; - boost::tie(protocol, m_auth, m_host, m_port, m_path) - = parse_url_components(url, ec); - TORRENT_ASSERT(!ec); - - if (!m_auth.empty()) - m_auth = base64encode(m_auth); - - m_server_string = "URL seed @ "; - m_server_string += m_host; - } - - void web_peer_connection::start() - { - set_upload_only(true); - if (is_disconnecting()) return; - peer_connection::start(); } void web_peer_connection::disconnect(error_code const& ec, int error) @@ -127,10 +95,7 @@ namespace libtorrent t->add_redundant_bytes(m_block_pos); peer_connection::disconnect(ec, error); - if (t) - { - t->disconnect_web_seed(this); - } + if (t) t->disconnect_web_seed(this); } boost::optional @@ -150,6 +115,8 @@ namespace libtorrent ret.block_index = (m_requests.front().start + m_block_pos - 1) / t->block_size(); else ret.block_index = (m_requests.front().start + m_block_pos) / t->block_size(); + TORRENT_ASSERT(ret.block_index < piece_block::invalid.block_index); + TORRENT_ASSERT(ret.piece_index < piece_block::invalid.piece_index); ret.full_block_bytes = t->block_size(); const int last_piece = t->torrent_file().num_pieces() - 1; @@ -159,20 +126,6 @@ namespace libtorrent return ret; } - void web_peer_connection::on_connected() - { - boost::shared_ptr t = associated_torrent().lock(); - TORRENT_ASSERT(t); - - // this is always a seed - incoming_have_all(); - - // it is always possible to request pieces - incoming_unchoke(); - - reset_recv_buffer(t->block_size() + 1024); - } - void web_peer_connection::write_request(peer_request const& r) { INVARIANT_CHECK; @@ -229,33 +182,11 @@ namespace libtorrent // assumed to be encoded in the torrent file request += using_proxy ? m_url : m_path; request += " HTTP/1.1\r\n"; - request += "Host: "; - request += m_host; - if (m_first_request && !m_ses.settings().user_agent.empty()) - { - request += "\r\nUser-Agent: "; - request += m_ses.settings().user_agent; - } - if (!m_auth.empty()) - { - request += "\r\nAuthorization: Basic "; - request += m_auth; - } - if (ps.type == proxy_settings::http_pw) - { - request += "\r\nProxy-Authorization: Basic "; - request += base64encode(ps.username + ":" + ps.password); - } - if (using_proxy) - { - request += "\r\nProxy-Connection: keep-alive"; - } + add_headers(request, ps, using_proxy); request += "\r\nRange: bytes="; request += to_string(size_type(r.piece) * info.piece_length() + r.start).elems; request += "-"; request += to_string(size_type(r.piece) * info.piece_length() + r.start + r.length - 1).elems; - if (m_first_request || using_proxy) - request += "\r\nConnection: keep-alive"; request += "\r\n\r\n"; m_first_request = false; m_file_requests.push_back(0); @@ -290,34 +221,13 @@ namespace libtorrent request += escape_path(path.c_str(), path.length()); } request += " HTTP/1.1\r\n"; - request += "Host: "; - request += m_host; - if (m_first_request && !m_ses.settings().user_agent.empty()) - { - request += "\r\nUser-Agent: "; - request += m_ses.settings().user_agent; - } - if (!m_auth.empty()) - { - request += "\r\nAuthorization: Basic "; - request += m_auth; - } - if (ps.type == proxy_settings::http_pw) - { - request += "\r\nProxy-Authorization: Basic "; - request += base64encode(ps.username + ":" + ps.password); - } - if (using_proxy) - { - request += "\r\nProxy-Connection: keep-alive"; - } + add_headers(request, ps, using_proxy); request += "\r\nRange: bytes="; request += to_string(f.offset).elems; request += "-"; request += to_string(f.offset + f.size - 1).elems; - if (m_first_request || using_proxy) - request += "\r\nConnection: keep-alive"; request += "\r\n\r\n"; + fprintf(stderr, "REQ: %s\n", request.c_str()); m_first_request = false; TORRENT_ASSERT(f.file_index >= 0); m_file_requests.push_back(f.file_index); @@ -713,45 +623,10 @@ namespace libtorrent void web_peer_connection::get_specific_peer_info(peer_info& p) const { - if (is_interesting()) p.flags |= peer_info::interesting; - if (is_choked()) p.flags |= peer_info::choked; - if (is_peer_interested()) p.flags |= peer_info::remote_interested; - if (has_peer_choked()) p.flags |= peer_info::remote_choked; - if (is_local()) p.flags |= peer_info::local_connection; - if (!is_connecting() && m_server_string.empty()) - p.flags |= peer_info::handshake; - if (is_connecting() && !is_queued()) p.flags |= peer_info::connecting; - if (is_queued()) p.flags |= peer_info::queued; - - p.client = m_server_string; + web_connection_base::get_specific_peer_info(p); + p.flags |= peer_info::local_connection; p.connection_type = peer_info::web_seed; } - bool web_peer_connection::in_handshake() const - { - return m_server_string.empty(); - } - - void web_peer_connection::on_sent(error_code const& error - , std::size_t bytes_transferred) - { - INVARIANT_CHECK; - - if (error) return; - m_statistics.sent_bytes(0, bytes_transferred); - } - - -#ifdef TORRENT_DEBUG - void web_peer_connection::check_invariant() const - { -/* - TORRENT_ASSERT(m_num_pieces == std::count( - m_have_piece.begin() - , m_have_piece.end() - , true)); -*/ } -#endif - }