From 26053e4b7694c9fe1c6b72d329bcd2fe3c3b978c Mon Sep 17 00:00:00 2001 From: Arvid Norberg Date: Thu, 30 Dec 2010 01:47:30 +0000 Subject: [PATCH] support adding torrents by url to a .torrent file --- ChangeLog | 3 +- bindings/python/src/session.cpp | 4 + docs/manual.rst | 14 ++- examples/client_test.cpp | 16 ++- include/libtorrent/add_torrent_params.hpp | 1 + include/libtorrent/torrent.hpp | 19 +++- src/metadata_transfer.cpp | 1 + src/session.cpp | 15 +++ src/session_impl.cpp | 13 ++- src/torrent.cpp | 132 ++++++++++++++++++++-- src/ut_metadata.cpp | 1 + 11 files changed, 201 insertions(+), 18 deletions(-) diff --git a/ChangeLog b/ChangeLog index b6f33ce03..04304efaf 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,4 +1,5 @@ - * supporr CDATA tags in xml parser + * support adding torrents by url to the .torrent file + * support CDATA tags in xml parser * use a python python dictionary for settings instead of session_settings object (in python bindings) * optimized metadata transfer (magnet link) startup time (shaved off about 1 second) * optimized swarm startup time (shaved off about 1 second) diff --git a/bindings/python/src/session.cpp b/bindings/python/src/session.cpp index 2dc4809e4..bc0fbfee9 100644 --- a/bindings/python/src/session.cpp +++ b/bindings/python/src/session.cpp @@ -187,6 +187,10 @@ namespace p.upload_mode = params["share_mode"]; if (params.has_key("override_resume_data")) p.override_resume_data = params["override_resume_data"]; + if (params.has_key("trackerid")) + p.trackerid = extract(params["trackerid"]); + if (params.has_key("url")) + p.url = extract(params["url"]); #ifndef BOOST_NO_EXCEPTIONS return s.add_torrent(p); diff --git a/docs/manual.rst b/docs/manual.rst index fadeafc29..2902bf686 100644 --- a/docs/manual.rst +++ b/docs/manual.rst @@ -384,6 +384,7 @@ add_torrent() std::vector const* file_priorities; bool share_mode; std::string trackerid; + std::string url; }; torrent_handle add_torrent(add_torrent_params const& params); @@ -396,9 +397,10 @@ object with all the parameters. The overload that does not take an ``error_code`` throws an exception on error and is not available when building without exception support. -The only mandatory parameter is ``save_path`` which is the directory where you +The only mandatory parameters are ``save_path`` which is the directory where you want the files to be saved. You also need to specify either the ``ti`` (the -torrent file) or ``info_hash`` (the info hash of the torrent). If you specify the +torrent file), the ``info_hash`` (the info hash of the torrent) or the ``url`` +(the URL to where to download the .torrent file from). If you specify the info-hash, the torrent file will be downloaded from peers, which requires them to support the metadata extension. For the metadata extension to work, libtorrent must be built with extensions enabled (``TORRENT_DISABLE_EXTENSIONS`` must not be @@ -410,6 +412,11 @@ If the torrent doesn't have a tracker, but relies on the DHT to find peers, the ``tracker_url`` can be 0, otherwise you might specify a tracker url that tracks this torrent. +If you specify a ``url``, the torrent will be set in ``downloading_metadata`` state +until the .torrent file has been downloaded. If there's any error while downloading, +the torrent will be stopped and the torrent error state (``torrent_status::error``) +will indicate what went wrong. The ``url`` may also refer to a magnet link. + If the torrent you are trying to add already exists in the session (is either queued for checking, being checked or downloading) ``add_torrent()`` will throw libtorrent_exception_ which derives from ``std::exception`` unless ``duplicate_is_error`` @@ -5448,6 +5455,9 @@ the ``add_torrent_params``, ``p``. See `add_torrent()`_. The overload that does not take an ``error_code`` throws an exception on error and is not available when building without exception support. +A simpler way to add a magnet link to a session is to pass in the +link through ``add_torrent_params::url`` argument to ``session::add_torrent()``. + For more information about magnet links, see `magnet links`_. make_magnet_uri() diff --git a/examples/client_test.cpp b/examples/client_test.cpp index c977d13bd..525339e68 100644 --- a/examples/client_test.cpp +++ b/examples/client_test.cpp @@ -797,7 +797,7 @@ int main(int argc, char* argv[]) { if (argc == 1) { - fprintf(stderr, "usage: client_test [OPTIONS] [TORRENT|MAGNETURL]\n\n" + fprintf(stderr, "usage: client_test [OPTIONS] [TORRENT|MAGNETURL|URL]\n\n" "OPTIONS:\n" " -f logs all events to the given file\n" " -o limits the number of simultaneous\n" @@ -848,7 +848,8 @@ int main(int argc, char* argv[]) " " "\n\n" "TORRENT is a path to a .torrent file\n" - "MAGNETURL is a magnet: url\n") + "MAGNETURL is a magnet link\n" + "URL is a url to a torrent file\n") ; return 0; } @@ -1108,16 +1109,19 @@ int main(int argc, char* argv[]) for (std::vector::iterator i = torrents.begin() , end(torrents.end()); i != end; ++i) { - // first see if this is a torrentless download - if (std::strstr(i->c_str(), "magnet:") == i->c_str()) + if (std::strstr(i->c_str(), "http://") == i->c_str() + || std::strstr(i->c_str(), "https://") == i->c_str() + || std::strstr(i->c_str(), "magnet:") == i->c_str()) { add_torrent_params p; p.share_mode = share_mode; p.save_path = save_path; p.storage_mode = (storage_mode_t)allocation_mode; - printf("adding MANGET link: %s\n", i->c_str()); + p.url = *i; + + printf("adding URL: %s\n", i->c_str()); error_code ec; - torrent_handle h = add_magnet_uri(ses, i->c_str(), p, ec); + torrent_handle h = ses.add_torrent(p, ec); if (ec) { fprintf(stderr, "%s\n", ec.message().c_str()); diff --git a/include/libtorrent/add_torrent_params.hpp b/include/libtorrent/add_torrent_params.hpp index aec9edf1c..73f90475c 100644 --- a/include/libtorrent/add_torrent_params.hpp +++ b/include/libtorrent/add_torrent_params.hpp @@ -85,6 +85,7 @@ namespace libtorrent std::vector const* file_priorities; bool share_mode; std::string trackerid; + std::string url; }; } diff --git a/include/libtorrent/torrent.hpp b/include/libtorrent/torrent.hpp index 70bc65be8..2df137333 100644 --- a/include/libtorrent/torrent.hpp +++ b/include/libtorrent/torrent.hpp @@ -80,6 +80,8 @@ POSSIBILITY OF SUCH DAMAGE. namespace libtorrent { + struct http_parser; + #if defined TORRENT_VERBOSE_LOGGING || defined TORRENT_LOGGING || defined TORRENT_ERROR_LOGGING struct logger; #endif @@ -108,7 +110,8 @@ namespace libtorrent public: torrent(aux::session_impl& ses, tcp::endpoint const& net_interface - , int block_size, int seq, add_torrent_params const& p); + , int block_size, int seq, add_torrent_params const& p + , sha1_hash const& info_hash); ~torrent(); #ifndef TORRENT_DISABLE_ENCRYPTION @@ -122,6 +125,8 @@ namespace libtorrent // starts the announce timer void start(); + void start_download_url(); + #ifndef TORRENT_DISABLE_EXTENSIONS void add_extension(boost::shared_ptr); void add_extension(boost::function(torrent*, void*)> const& ext @@ -176,6 +181,7 @@ namespace libtorrent , peer_request p); void on_disk_cache_complete(int ret, disk_io_job const& j); + void set_progress_ppm(int p) { m_progress_ppm = p; } struct read_piece_struct { boost::shared_array piece_data; @@ -761,6 +767,9 @@ namespace libtorrent // a return value of false indicates an error bool set_metadata(char const* metadata_buf, int metadata_size); + void on_torrent_download(error_code const& ec, http_parser const& parser + , char const* data, int size); + int sequence_number() const { return m_sequence_number; } bool seed_mode() const { return m_seed_mode; } @@ -964,6 +973,14 @@ namespace libtorrent std::string m_save_path; + // if we don't have the metadata, this is a url to + // the torrent file + std::string m_url; + + // this is used as temporary storage while downloading + // the .torrent file from m_url + std::vector m_torrent_file_buf; + // each bit represents a piece. a set bit means // the piece has had its hash verified. This // is only used in seed mode (when m_seed_mode diff --git a/src/metadata_transfer.cpp b/src/metadata_transfer.cpp index f13bb5606..57a8d6c11 100644 --- a/src/metadata_transfer.cpp +++ b/src/metadata_transfer.cpp @@ -198,6 +198,7 @@ namespace libtorrent { namespace { m_metadata_progress += received; m_metadata_size = total_size; + m_torrent.set_progress_ppm(boost::int64_t(m_metadata_progress) * 1000000 / m_metadata_size); } void on_piece_pass(int) diff --git a/src/session.cpp b/src/session.cpp index e5a30a851..5331906fd 100644 --- a/src/session.cpp +++ b/src/session.cpp @@ -72,6 +72,7 @@ POSSIBILITY OF SUCH DAMAGE. #include "libtorrent/kademlia/dht_tracker.hpp" #include "libtorrent/natpmp.hpp" #include "libtorrent/upnp.hpp" +#include "libtorrent/magnet_uri.hpp" using boost::shared_ptr; using boost::weak_ptr; @@ -481,6 +482,13 @@ namespace libtorrent #ifndef BOOST_NO_EXCEPTIONS torrent_handle session::add_torrent(add_torrent_params const& params) { + if (string_begins_no_case("magnet:", params.url.c_str())) + { + add_torrent_params p(params); + p.url.clear(); + return add_magnet_uri(*this, params.url, p); + } + error_code ec; TORRENT_SYNC_CALL_RET2(torrent_handle, add_torrent, params, ec); if (ec) throw libtorrent_exception(ec); @@ -490,6 +498,13 @@ namespace libtorrent torrent_handle session::add_torrent(add_torrent_params const& params, error_code& ec) { + if (string_begins_no_case("magnet:", params.url.c_str())) + { + add_torrent_params p(params); + p.url.clear(); + return add_magnet_uri(*this, params.url, p, ec); + } + TORRENT_SYNC_CALL_RET2(torrent_handle, add_torrent, params, ec); return r; } diff --git a/src/session_impl.cpp b/src/session_impl.cpp index d98f4c2d0..1f631afcc 100644 --- a/src/session_impl.cpp +++ b/src/session_impl.cpp @@ -3453,7 +3453,18 @@ namespace aux { // figure out the info hash of the torrent sha1_hash const* ih = 0; + sha1_hash tmp; if (params.ti) ih = ¶ms.ti->info_hash(); + else if (!params.url.empty()) + { + // in order to avoid info-hash collisions, for + // torrents where we don't have an info-hash, but + // just a URL, set the temporary info-hash to the + // hash of the URL. This will be changed once we + // have the actual .torrent file + tmp = hasher(¶ms.url[0], params.url.size()).final(); + ih = &tmp; + } else ih = ¶ms.info_hash; // is the torrent already active? @@ -3476,7 +3487,7 @@ namespace aux { } torrent_ptr.reset(new torrent(*this, m_listen_interface - , 16 * 1024, queue_pos, params)); + , 16 * 1024, queue_pos, params, *ih)); torrent_ptr->start(); #ifndef TORRENT_DISABLE_EXTENSIONS diff --git a/src/torrent.cpp b/src/torrent.cpp index c1dcd0042..077cc4992 100644 --- a/src/torrent.cpp +++ b/src/torrent.cpp @@ -78,6 +78,8 @@ POSSIBILITY OF SUCH DAMAGE. #include "libtorrent/kademlia/dht_tracker.hpp" #include "libtorrent/peer_info.hpp" #include "libtorrent/enum_net.hpp" +#include "libtorrent/http_connection.hpp" +#include "libtorrent/gzip.hpp" // for inflate_gzip #ifdef TORRENT_USE_OPENSSL #include "libtorrent/ssl_stream.hpp" @@ -321,18 +323,20 @@ namespace libtorrent , tcp::endpoint const& net_interface , int block_size , int seq - , add_torrent_params const& p) + , add_torrent_params const& p + , sha1_hash const& info_hash) : m_policy(this) , m_total_uploaded(0) , m_total_downloaded(0) , m_started(time_now()) - , m_torrent_file(p.ti ? p.ti : new torrent_info(p.info_hash)) + , m_torrent_file(p.ti ? p.ti : new torrent_info(info_hash)) , m_storage(0) , m_tracker_timer(ses.m_io_service) , m_ses(ses) , m_trackers(m_torrent_file->trackers()) - , m_save_path(complete(p.save_path)) , m_trackerid(p.trackerid) + , m_save_path(complete(p.save_path)) + , m_url(p.url) , m_storage_constructor(p.storage) , m_ratio(0.f) , m_available_free_upload(0) @@ -427,6 +431,7 @@ namespace libtorrent INVARIANT_CHECK; if (p.name && !p.ti) m_name.reset(new std::string(p.name)); + if (!m_name && !m_url.empty()) m_name.reset(new std::string(m_url)); if (p.tracker_url && std::strlen(p.tracker_url) > 0) { @@ -437,6 +442,95 @@ namespace libtorrent } } + // since this download is not bottled, this callback will + // be called every time we receive another piece of the + // .torrent file + void torrent::on_torrent_download(error_code const& ec + , http_parser const& parser + , char const* data, int size) + { + if (ec && ec != asio::error::eof) + { + set_error(ec, m_url); + pause(); + return; + } + if (size > 0) + { + m_torrent_file_buf.insert(m_torrent_file_buf.end(), data, data + size); + if (parser.content_length() > 0) + set_progress_ppm(boost::int64_t(m_torrent_file_buf.size()) + * 1000000 / parser.content_length()); + } + + if (!ec) return; + + std::string const& encoding = parser.header("content-encoding"); + if ((encoding == "gzip" || encoding == "x-gzip") && m_torrent_file_buf.size()) + { + std::vector buf; + std::string error; + if (inflate_gzip(&m_torrent_file_buf[0], m_torrent_file_buf.size() + , buf, 4 * 1024 * 1024, error)) + { + set_error(errors::http_failed_decompress, m_url); + pause(); + std::vector().swap(m_torrent_file_buf); + return; + } + m_torrent_file_buf.swap(buf); + } + + // we're done! + error_code e; + intrusive_ptr tf(new torrent_info( + &m_torrent_file_buf[0], m_torrent_file_buf.size(), e)); + if (e) + { + set_error(e, m_url); + pause(); + std::vector().swap(m_torrent_file_buf); + return; + } + std::vector().swap(m_torrent_file_buf); + + // update our torrent_info object and move the + // torrent from the old info-hash to the new one + // as we replace the torrent_info object +#ifdef TORRENT_DEBUG + int num_torrents = m_ses.m_torrents.size(); +#endif + m_ses.m_torrents.erase(m_torrent_file->info_hash()); + m_torrent_file = tf; + m_ses.m_torrents.insert(std::make_pair(m_torrent_file->info_hash(), shared_from_this())); + + TORRENT_ASSERT(num_torrents == m_ses.m_torrents.size()); + + // TODO: if the user added any trackers while downloading the + // .torrent file, they are overwritten. Merge them into the + // new tracker list + m_trackers = m_torrent_file->trackers(); + +#ifndef TORRENT_DISABLE_ENCRYPTION + hasher h; + h.update("req2", 4); + h.update((char*)&m_torrent_file->info_hash()[0], 20); + m_obfuscated_hash = h.final(); +#endif + + if (m_ses.m_alerts.should_post()) + { + m_ses.m_alerts.post_alert(metadata_received_alert( + get_handle())); + } + + set_state(torrent_status::downloading); + + m_override_resume_data = true; + init(); + announce_with_tracker(); + } + void torrent::start() { TORRENT_ASSERT(m_ses.is_network_thread()); @@ -472,10 +566,15 @@ namespace libtorrent } } - // we need to start announcing since we don't have any - // metadata. To receive peers to ask for it. - if (m_torrent_file->is_valid()) + if (!m_torrent_file->is_valid() && !m_url.empty()) { + // we need to download the .torrent file from m_url + start_download_url(); + } + else if (m_torrent_file->is_valid()) + { + // we need to start announcing since we don't have any + // metadata. To receive peers to ask for it. init(); } else @@ -485,6 +584,18 @@ namespace libtorrent } } + void torrent::start_download_url() + { + TORRENT_ASSERT(!m_url.empty()); + TORRENT_ASSERT(!m_torrent_file->is_valid()); + boost::shared_ptr conn( + new http_connection(m_ses.m_io_service, m_ses.m_half_open + , boost::bind(&torrent::on_torrent_download, shared_from_this() + , _1, _2, _3, _4), false)); + conn->get(m_url); + set_state(torrent_status::downloading_metadata); + } + #ifndef TORRENT_DISABLE_DHT bool torrent::should_announce_dht() const { @@ -920,7 +1031,6 @@ namespace libtorrent // ans also in the case of share mode, we need to update the priorities update_piece_priorities(); - std::vector const& web_seeds = m_torrent_file->web_seeds(); m_web_seeds.insert(m_web_seeds.end(), web_seeds.begin(), web_seeds.end()); @@ -5518,10 +5628,18 @@ namespace libtorrent m_ses.m_auto_manage_time_scaler = 2; m_error = error_code(); m_error_file.clear(); + + // if we haven't downloaded the metadata from m_url, try again + if (!m_url.empty() && !m_torrent_file->is_valid()) + { + start_download_url(); + return; + } // if the error happened during initialization, try again now if (!m_storage) init(); if (!checking_files && should_check_files()) queue_torrent_check(); + } void torrent::set_error(error_code const& ec, std::string const& error_file) diff --git a/src/ut_metadata.cpp b/src/ut_metadata.cpp index f3bab0d44..b7c689b41 100644 --- a/src/ut_metadata.cpp +++ b/src/ut_metadata.cpp @@ -162,6 +162,7 @@ namespace libtorrent { namespace { m_metadata_progress += received; m_metadata_size = total_size; + m_torrent.set_progress_ppm(boost::int64_t(m_metadata_progress) * 1000000 / m_metadata_size); } void on_piece_pass(int)