support adding torrents by url to a .torrent file

ChangeLog

@@ -1,4 +1,5 @@
-	* supporr CDATA tags in xml parser
+	* support adding torrents by url to the .torrent file
+	* support CDATA tags in xml parser
 	* use a python python dictionary for settings instead of session_settings object (in python bindings)
 	* optimized metadata transfer (magnet link) startup time (shaved off about 1 second)
 	* optimized swarm startup time (shaved off about 1 second)

bindings/python/src/session.cpp

@@ -187,6 +187,10 @@ namespace
             p.upload_mode = params["share_mode"];
         if (params.has_key("override_resume_data"))
             p.override_resume_data = params["override_resume_data"];
+        if (params.has_key("trackerid"))
+            p.trackerid = extract<std::string>(params["trackerid"]);
+        if (params.has_key("url"))
+            p.url = extract<std::string>(params["url"]);
 
 #ifndef BOOST_NO_EXCEPTIONS
         return s.add_torrent(p);

docs/manual.rst

@@ -384,6 +384,7 @@ add_torrent()
 			std::vector<boost::uint8_t> const* file_priorities;
 			bool share_mode;
 			std::string trackerid;
+			std::string url;
 		};
 
 		torrent_handle add_torrent(add_torrent_params const& params);
@@ -396,9 +397,10 @@ object with all the parameters.
 The overload that does not take an ``error_code`` throws an exception on
 error and is not available when building without exception support.
 
-The only mandatory parameter is ``save_path`` which is the directory where you
+The only mandatory parameters are ``save_path`` which is the directory where you
 want the files to be saved. You also need to specify either the ``ti`` (the
-torrent file) or ``info_hash`` (the info hash of the torrent). If you specify the
+torrent file), the ``info_hash`` (the info hash of the torrent) or the ``url``
+(the URL to where to download the .torrent file from). If you specify the
 info-hash, the torrent file will be downloaded from peers, which requires them to
 support the metadata extension. For the metadata extension to work, libtorrent must
 be built with extensions enabled (``TORRENT_DISABLE_EXTENSIONS`` must not be
@@ -410,6 +412,11 @@ If the torrent doesn't have a tracker, but relies on the DHT to find peers, the
 ``tracker_url`` can be 0, otherwise you might specify a tracker url that tracks this
 torrent.
 
+If you specify a ``url``, the torrent will be set in ``downloading_metadata`` state
+until the .torrent file has been downloaded. If there's any error while downloading,
+the torrent will be stopped and the torrent error state (``torrent_status::error``)
+will indicate what went wrong. The ``url`` may also refer to a magnet link.
+
 If the torrent you are trying to add already exists in the session (is either queued
 for checking, being checked or downloading) ``add_torrent()`` will throw
 libtorrent_exception_ which derives from ``std::exception`` unless ``duplicate_is_error``
@@ -5448,6 +5455,9 @@ the ``add_torrent_params``, ``p``. See `add_torrent()`_.
 The overload that does not take an ``error_code`` throws an exception on
 error and is not available when building without exception support.
 
+A simpler way to add a magnet link to a session is to pass in the
+link through ``add_torrent_params::url`` argument to ``session::add_torrent()``.
+
 For more information about magnet links, see `magnet links`_.
 
 make_magnet_uri()

examples/client_test.cpp

@@ -797,7 +797,7 @@ int main(int argc, char* argv[])
 {
 	if (argc == 1)
 	{
-		fprintf(stderr, "usage: client_test [OPTIONS] [TORRENT|MAGNETURL]\n\n"
+		fprintf(stderr, "usage: client_test [OPTIONS] [TORRENT|MAGNETURL|URL]\n\n"
 			"OPTIONS:\n"
 			"  -f <log file>         logs all events to the given file\n"
 			"  -o <limit>            limits the number of simultaneous\n"
@@ -848,7 +848,8 @@ int main(int argc, char* argv[])
 			"  "
 			"\n\n"
 			"TORRENT is a path to a .torrent file\n"
-			"MAGNETURL is a magnet: url\n")
+			"MAGNETURL is a magnet link\n"
+			"URL is a url to a torrent file\n")
 			;
 		return 0;
 	}
@@ -1108,16 +1109,19 @@ int main(int argc, char* argv[])
 	for (std::vector<std::string>::iterator i = torrents.begin()
 		, end(torrents.end()); i != end; ++i)
 	{
-		// first see if this is a torrentless download
-		if (std::strstr(i->c_str(), "magnet:") == i->c_str())
+		if (std::strstr(i->c_str(), "http://") == i->c_str()
+			|| std::strstr(i->c_str(), "https://") == i->c_str()
+			|| std::strstr(i->c_str(), "magnet:") == i->c_str())
 		{
 			add_torrent_params p;
 			p.share_mode = share_mode;
 			p.save_path = save_path;
 			p.storage_mode = (storage_mode_t)allocation_mode;
-			printf("adding MANGET link: %s\n", i->c_str());
+			p.url = *i;
+
+			printf("adding URL: %s\n", i->c_str());
 			error_code ec;
-			torrent_handle h = add_magnet_uri(ses, i->c_str(), p, ec);
+			torrent_handle h = ses.add_torrent(p, ec);
 			if (ec)
 			{
 				fprintf(stderr, "%s\n", ec.message().c_str());

include/libtorrent/add_torrent_params.hpp

@@ -85,6 +85,7 @@ namespace libtorrent
 		std::vector<boost::uint8_t> const* file_priorities;
 		bool share_mode;
 		std::string trackerid;
+		std::string url;
 	};
 }
 

include/libtorrent/torrent.hpp

@@ -80,6 +80,8 @@ POSSIBILITY OF SUCH DAMAGE.
 
 namespace libtorrent
 {
+	struct http_parser;
+
 #if defined TORRENT_VERBOSE_LOGGING || defined TORRENT_LOGGING || defined TORRENT_ERROR_LOGGING
 	struct logger;
 #endif
@@ -108,7 +110,8 @@ namespace libtorrent
 	public:
 
 		torrent(aux::session_impl& ses, tcp::endpoint const& net_interface
-			, int block_size, int seq, add_torrent_params const& p);
+			, int block_size, int seq, add_torrent_params const& p
+			, sha1_hash const& info_hash);
 		~torrent();
 
 #ifndef TORRENT_DISABLE_ENCRYPTION
@@ -122,6 +125,8 @@ namespace libtorrent
 		// starts the announce timer
 		void start();
 
+		void start_download_url();
+
 #ifndef TORRENT_DISABLE_EXTENSIONS
 		void add_extension(boost::shared_ptr<torrent_plugin>);
 		void add_extension(boost::function<boost::shared_ptr<torrent_plugin>(torrent*, void*)> const& ext
@@ -176,6 +181,7 @@ namespace libtorrent
 			, peer_request p);
 		void on_disk_cache_complete(int ret, disk_io_job const& j);
 
+		void set_progress_ppm(int p) { m_progress_ppm = p; }
 		struct read_piece_struct
 		{
 			boost::shared_array<char> piece_data;
@@ -761,6 +767,9 @@ namespace libtorrent
 		// a return value of false indicates an error
 		bool set_metadata(char const* metadata_buf, int metadata_size);
 
+		void on_torrent_download(error_code const& ec, http_parser const& parser
+			, char const* data, int size);
+
 		int sequence_number() const { return m_sequence_number; }
 
 		bool seed_mode() const { return m_seed_mode; }
@@ -964,6 +973,14 @@ namespace libtorrent
 
 		std::string m_save_path;
 
+		// if we don't have the metadata, this is a url to
+		// the torrent file
+		std::string m_url;
+
+		// this is used as temporary storage while downloading
+		// the .torrent file from m_url
+		std::vector<char> m_torrent_file_buf;
+
 		// each bit represents a piece. a set bit means
 		// the piece has had its hash verified. This
 		// is only used in seed mode (when m_seed_mode

src/metadata_transfer.cpp

@@ -198,6 +198,7 @@ namespace libtorrent { namespace
 		{
 			m_metadata_progress += received;
 			m_metadata_size = total_size;
+			m_torrent.set_progress_ppm(boost::int64_t(m_metadata_progress) * 1000000 / m_metadata_size);
 		}
 
 		void on_piece_pass(int)

src/session.cpp

@@ -72,6 +72,7 @@ POSSIBILITY OF SUCH DAMAGE.
 #include "libtorrent/kademlia/dht_tracker.hpp"
 #include "libtorrent/natpmp.hpp"
 #include "libtorrent/upnp.hpp"
+#include "libtorrent/magnet_uri.hpp"
 
 using boost::shared_ptr;
 using boost::weak_ptr;
@@ -481,6 +482,13 @@ namespace libtorrent
 #ifndef BOOST_NO_EXCEPTIONS
 	torrent_handle session::add_torrent(add_torrent_params const& params)
 	{
+		if (string_begins_no_case("magnet:", params.url.c_str()))
+		{
+			add_torrent_params p(params);
+			p.url.clear();
+			return add_magnet_uri(*this, params.url, p);
+		}
+
 		error_code ec;
 		TORRENT_SYNC_CALL_RET2(torrent_handle, add_torrent, params, ec);
 		if (ec) throw libtorrent_exception(ec);
@@ -490,6 +498,13 @@ namespace libtorrent
 
 	torrent_handle session::add_torrent(add_torrent_params const& params, error_code& ec)
 	{
+		if (string_begins_no_case("magnet:", params.url.c_str()))
+		{
+			add_torrent_params p(params);
+			p.url.clear();
+			return add_magnet_uri(*this, params.url, p, ec);
+		}
+
 		TORRENT_SYNC_CALL_RET2(torrent_handle, add_torrent, params, ec);
 		return r;
 	}

src/session_impl.cpp

@@ -3453,7 +3453,18 @@ namespace aux {
 		
 		// figure out the info hash of the torrent
 		sha1_hash const* ih = 0;
+		sha1_hash tmp;
 		if (params.ti) ih = &params.ti->info_hash();
+		else if (!params.url.empty())
+		{
+			// in order to avoid info-hash collisions, for
+			// torrents where we don't have an info-hash, but
+			// just a URL, set the temporary info-hash to the
+			// hash of the URL. This will be changed once we
+			// have the actual .torrent file
+			tmp = hasher(&params.url[0], params.url.size()).final();
+			ih = &tmp;
+		}
 		else ih = &params.info_hash;
 
 		// is the torrent already active?
@@ -3476,7 +3487,7 @@ namespace aux {
 		}
 
 		torrent_ptr.reset(new torrent(*this, m_listen_interface
-			, 16 * 1024, queue_pos, params));
+			, 16 * 1024, queue_pos, params, *ih));
 		torrent_ptr->start();
 
 #ifndef TORRENT_DISABLE_EXTENSIONS

src/torrent.cpp

@@ -78,6 +78,8 @@ POSSIBILITY OF SUCH DAMAGE.
 #include "libtorrent/kademlia/dht_tracker.hpp"
 #include "libtorrent/peer_info.hpp"
 #include "libtorrent/enum_net.hpp"
+#include "libtorrent/http_connection.hpp"
+#include "libtorrent/gzip.hpp" // for inflate_gzip
 
 #ifdef TORRENT_USE_OPENSSL
 #include "libtorrent/ssl_stream.hpp"
@@ -321,18 +323,20 @@ namespace libtorrent
 		, tcp::endpoint const& net_interface
 		, int block_size
 		, int seq
-		, add_torrent_params const& p)
+		, add_torrent_params const& p
+		, sha1_hash const& info_hash)
 		: m_policy(this)
 		, m_total_uploaded(0)
 		, m_total_downloaded(0)
 		, m_started(time_now())
-		, m_torrent_file(p.ti ? p.ti : new torrent_info(p.info_hash))
+		, m_torrent_file(p.ti ? p.ti : new torrent_info(info_hash))
 		, m_storage(0)
 		, m_tracker_timer(ses.m_io_service)
 		, m_ses(ses)
 		, m_trackers(m_torrent_file->trackers())
-		, m_save_path(complete(p.save_path))
 		, m_trackerid(p.trackerid)
+		, m_save_path(complete(p.save_path))
+		, m_url(p.url)
 		, m_storage_constructor(p.storage)
 		, m_ratio(0.f)
 		, m_available_free_upload(0)
@@ -427,6 +431,7 @@ namespace libtorrent
 		INVARIANT_CHECK;
 
 		if (p.name && !p.ti) m_name.reset(new std::string(p.name));
+		if (!m_name && !m_url.empty()) m_name.reset(new std::string(m_url));
 
 		if (p.tracker_url && std::strlen(p.tracker_url) > 0)
 		{
@@ -437,6 +442,95 @@ namespace libtorrent
 		}
 	}
 
+	// since this download is not bottled, this callback will
+	// be called every time we receive another piece of the
+	// .torrent file
+	void torrent::on_torrent_download(error_code const& ec
+		, http_parser const& parser
+		, char const* data, int size)
+	{
+		if (ec && ec != asio::error::eof)
+		{
+			set_error(ec, m_url);
+			pause();
+			return;
+		}
+		if (size > 0)
+		{
+			m_torrent_file_buf.insert(m_torrent_file_buf.end(), data, data + size);
+			if (parser.content_length() > 0)
+				set_progress_ppm(boost::int64_t(m_torrent_file_buf.size())
+					* 1000000 / parser.content_length());
+		}
+
+		if (!ec) return;
+
+		std::string const& encoding = parser.header("content-encoding");
+		if ((encoding == "gzip" || encoding == "x-gzip") && m_torrent_file_buf.size())
+		{
+			std::vector<char> buf;
+			std::string error;
+			if (inflate_gzip(&m_torrent_file_buf[0], m_torrent_file_buf.size()
+				, buf, 4 * 1024 * 1024, error))
+			{
+				set_error(errors::http_failed_decompress, m_url);
+				pause();
+				std::vector<char>().swap(m_torrent_file_buf);
+				return;
+			}
+			m_torrent_file_buf.swap(buf);
+		}
+
+		// we're done!
+		error_code e;
+		intrusive_ptr<torrent_info> tf(new torrent_info(
+			&m_torrent_file_buf[0], m_torrent_file_buf.size(), e));
+		if (e)
+		{
+			set_error(e, m_url);
+			pause();
+			std::vector<char>().swap(m_torrent_file_buf);
+			return;
+		}
+		std::vector<char>().swap(m_torrent_file_buf);
+		
+		// update our torrent_info object and move the
+		// torrent from the old info-hash to the new one
+		// as we replace the torrent_info object
+#ifdef TORRENT_DEBUG
+		int num_torrents = m_ses.m_torrents.size();
+#endif
+		m_ses.m_torrents.erase(m_torrent_file->info_hash());
+		m_torrent_file = tf;
+		m_ses.m_torrents.insert(std::make_pair(m_torrent_file->info_hash(), shared_from_this()));
+
+		TORRENT_ASSERT(num_torrents == m_ses.m_torrents.size());
+
+		// TODO: if the user added any trackers while downloading the
+		// .torrent file, they are overwritten. Merge them into the
+		// new tracker list
+		m_trackers = m_torrent_file->trackers();
+
+#ifndef TORRENT_DISABLE_ENCRYPTION
+		hasher h;
+		h.update("req2", 4);
+		h.update((char*)&m_torrent_file->info_hash()[0], 20);
+		m_obfuscated_hash = h.final();
+#endif
+
+		if (m_ses.m_alerts.should_post<metadata_received_alert>())
+		{
+			m_ses.m_alerts.post_alert(metadata_received_alert(
+				get_handle()));
+		}
+
+		set_state(torrent_status::downloading);
+
+		m_override_resume_data = true;
+		init();
+		announce_with_tracker();
+	}
+
 	void torrent::start()
 	{
 		TORRENT_ASSERT(m_ses.is_network_thread());
@@ -472,10 +566,15 @@ namespace libtorrent
 			}
 		}
 
+		if (!m_torrent_file->is_valid() && !m_url.empty())
+		{
+			// we need to download the .torrent file from m_url
+			start_download_url();
+		}
+		else if (m_torrent_file->is_valid())
+		{
 			// we need to start announcing since we don't have any
 			// metadata. To receive peers to ask for it.
-		if (m_torrent_file->is_valid())
-		{
 			init();
 		}
 		else
@@ -485,6 +584,18 @@ namespace libtorrent
 		}
 	}
 
+	void torrent::start_download_url()
+	{
+		TORRENT_ASSERT(!m_url.empty());
+		TORRENT_ASSERT(!m_torrent_file->is_valid());
+		boost::shared_ptr<http_connection> conn(
+			new http_connection(m_ses.m_io_service, m_ses.m_half_open
+				, boost::bind(&torrent::on_torrent_download, shared_from_this()
+					, _1, _2, _3, _4), false));
+		conn->get(m_url);
+		set_state(torrent_status::downloading_metadata);
+	}
+
 #ifndef TORRENT_DISABLE_DHT
 	bool torrent::should_announce_dht() const
 	{
@@ -920,7 +1031,6 @@ namespace libtorrent
 		// ans also in the case of share mode, we need to update the priorities
 		update_piece_priorities();
 
-
 		std::vector<web_seed_entry> const& web_seeds = m_torrent_file->web_seeds();
 		m_web_seeds.insert(m_web_seeds.end(), web_seeds.begin(), web_seeds.end());
 
@@ -5518,10 +5628,18 @@ namespace libtorrent
 			m_ses.m_auto_manage_time_scaler = 2;
 		m_error = error_code();
 		m_error_file.clear();
+
+		// if we haven't downloaded the metadata from m_url, try again
+		if (!m_url.empty() && !m_torrent_file->is_valid())
+		{
+			start_download_url();
+			return;
+		}
 		// if the error happened during initialization, try again now
 		if (!m_storage) init();
 		if (!checking_files && should_check_files())
 			queue_torrent_check();
+
 	}
 
 	void torrent::set_error(error_code const& ec, std::string const& error_file)

src/ut_metadata.cpp

@@ -162,6 +162,7 @@ namespace libtorrent { namespace
 		{
 			m_metadata_progress += received;
 			m_metadata_size = total_size;
+			m_torrent.set_progress_ppm(boost::int64_t(m_metadata_progress) * 1000000 / m_metadata_size);
 		}
 
 		void on_piece_pass(int)