Merge pull request #488 from arvidn/tutorial-1.1

extend the tutorial

.travis.yml

@@ -91,6 +91,9 @@ script:
 
   - cd ../examples
   - bjam --hash -j3 warnings-as-errors=on variant=$variant $toolset link=shared
+  - if [[ lang == "cpp11" ]]; then
+    bjam --hash -j3 warnings-as-errors=on variant=$variant $toolset link=shared bt-get bt-get2;
+    fi
   - cd ..
 
 # build libtorrent separately and install it in a temporary (well known) dir

appveyor.yml

@@ -117,6 +117,9 @@ build_script:
 - if not defined x64 (
   cd %ROOT_DIRECTORY%\examples
   & b2.exe --hash -j2 %compiler% variant=%variant% linkflags=%linkflags% include=%include% link=shared
+  & if %compiler% == msvc-14.0 ( b2.exe --hash -j2 %compiler% variant=%variant% linkflags=%linkflags% include=%include% link=shared bt-get bt-get2 )
+  & cd %ROOT_DIRECTORY%\examples
+  & b2.exe --hash -j2 %compiler% variant=%variant% linkflags=%linkflags% include=%include% link=shared
   & cd %ROOT_DIRECTORY%\bindings\python
   & b2.exe --hash -j2 %compiler% stage_module install-dependencies=on variant=%variant% libtorrent-link=shared linkflags=%linkflags% include=%include%
   & python test.py

docs/gen_reference_doc.py

@@ -149,6 +149,7 @@ def is_visible(desc):
 	return True
 
 def highlight_signature(s):
+	s = s.replace('TORRENT_OVERRIDE', 'override').replace('TORRENT_FINAL', 'final')
 	name = s.split('(', 1)
 	name2 = name[0].split(' ')
 	if len(name2[-1]) == 0: return s
@@ -273,7 +274,8 @@ def parse_class(lno, lines, filename):
 		state = 'private'
 		class_type = 'class'
 
-	name = decl.split(':')[0].replace('class ', '').replace('struct ', '').replace('TORRENT_FINAL', '').strip()
+	decl = decl.replace('TORRENT_FINAL', 'final')
+	name = decl.split(':')[0].replace('class ', '').replace('struct ', '').replace('final', '').strip()
 
 
 	while lno < len(lines):

docs/tutorial.rst

@@ -70,46 +70,165 @@ completes downloading, we can poll the session for alerts periodically and print
 them out, as well as listening for the torrent_finished_alert_, which is posted
 when a torrent completes.
 
+.. include:: ../examples/bt-get.cpp
+	:code: c++
+	:tab-width: 2
+	:start-after: */
+
+alert masks
+-----------
+
+The output from this program will be quite verbose, which is probably a good
+starting point to get some understanding of what's going on. Alerts are
+categorized into alert categories. Each category can be enabled and disabled
+independently via the *alert mask*.
+
+The alert mask is a configuration option offered by libtorrent. There are many
+configuration options, see settings_pack_. The alert_mask_ setting is an integer
+of the `category flags`_ ORed together.
+
+For instance, to only see the most pertinent alerts, the session can be
+constructed like this:
+
 .. code:: c++
 
-	#include <iostream>
+	lt::settings_pack pack;
+	pack.set_int(lt::settings_pack::alert_mask
+		, lt::alert::error_notification
+		| lt::alert::storage_notification
+		| lt::alert::status_notification);
 
-	#include <libtorrent/session.hpp>
+	lt::session ses(pack);
-	#include <libtorrent/add_torrent_params.hpp>
-	#include <libtorrent/torrent_handle.hpp>
-	#include <libtorrent/alert_types.hpp>
 
-	namespace lt = libtorrent;
+Configuration options can be updated after the session is started by calling
-	int main(int argc, char const* argv[])
+`apply_settings()`_. Some settings are best set before starting the session
+though, like listen_interfaces_, to avoid race conditions. If you start the
+session with the default settings and then immediately change them, there will
+still be a window where the default settings apply.
+
+Changing the settings may trigger listen sockets to close and re-open and
+NAT-PMP, UPnP updates to be sent. For this reason, it's typically a good idea
+to batch settings updates into a single call.
+
+session destruction
+-------------------
+
+The session destructor is blocking by default. When shutting down, trackers
+will need to be contacted to stop torrents and other outstanding operations
+need to be cancelled. Shutting down can sometimes take several seconds,
+primarily because of trackers that are unresponsive (and time out) and also
+DNS servers that are unresponsive. DNS lookups are especially difficult to
+abort when stalled.
+
+In order to be able to start destruction an wait for it asynchronously, one
+can call `session::abort()`_.
+
+This call returns a session_proxy_ object, which is a handle keeping the session
+state alive while destructing it. It deliberately does not provide any of the
+session operations, since it's shutting down.
+
+After having a session_proxy_ object, the session destructor does not block.
+However, the session_proxy_ destructor *will*.
+
+This can be used to shut down multiple sessions or other parts of the
+application in parallel.
+
+asynchronous operations
+-----------------------
+
+Essentially any call to a member function of session_ or torrent_handle_ that
+returns a value is a blocking synchronous call. Meaning it will post a message
+to the main libtorrent thread and wait for a response. Such calls may be
+expensive, and in applications where stalls should be avoided (such as user
+interface threads), blocking calls should be avoided.
+
+In the example above, session::add_torrent() returns a torrent_handle_ and is
+thus blocking. For higher efficiency, `async_add_torrent()`_ will post a message
+to the main thread to add a torrent, and post the resulting torrent_handle_ back
+in an alert (add_torrent_alert_). This is especially useful when adding a lot
+of torrents in quick succession, as there's no stall in between calls.
+
+In the example above, we don't actually use the torrent_handle_ for anything, so
+converting it to use `async_add_torrent()`_ is just a matter of replacing the
+`add_torrent()`_ call with `async_add_torrent()`_.
+
+torrent_status_updates
+----------------------
+
+To get updates to the status of torrents, call `post_torrent_updates()`_ on the
+session object. This will cause libtorrent to post a state_update_alert_
+containing torrent_status_ objects for all torrents whose status has *changed*
+since the last call to `post_torrent_updates()`_.
+
+The state_update_alert_ looks something like this:
+
+.. code:: c++
+
+	struct state_update_alert : alert
 	{
-		if (argc != 2) {
+		virtual std::string message() const;
-			std::cerr << "usage: " << argv[0] << " <magnet-url>" << std::endl;
+		std::vector<torrent_status> status;
-			return 1;
+	};
-		}
-		lt::session ses;
 
-		lt::add_torrent_params atp;
+The ``status`` field only contains the torrent_status_ for torrents with
-		atp.url = argv[1];
+updates since the last call. It may be empty if no torrent has updated its
-		atp.save_path = "."; // save in current dir
+state. This feature is critical for scalability_.
-		lt::torrent_handle h = ses.add_torrent(atp);
 
-		bool done = false;
+See the torrent_status_ object for more information on what is in there.
-		while (!done) {
+Perhaps the most interesting fields are ``total_payload_download``,
-			std::vector<lt::alert*> alerts;
+``total_payload_upload``, ``num_peers`` and ``state``.
-			ses.pop_alerts(&alerts);
 
-			for (lt::alert const* a : alerts) {
+resuming torrents
-				std::cout << a->message() << std::endl;
+-----------------
-				if (lt::alert_cast<lt::torrent_finished_alert>(a)) {
-					done = true;
-				}
-			}
-		}
-	}
 
-*TODO* cover async_add_torrent()
+Since bittorrent downloads pieces of files in random order, it's not trivial to
-*TODO* cover post_torrent_updates()
+resume a partial download. When resuming a download, the bittorrent engine must
-*TODO* cover save_resume_data()
+restore the state of the downloading torrent, specifically, which parts of the
+file(s) are downloaded. There are two approaches to doing this:
+
+1. read every piece of the downloaded files from disk and compare it against the
+   expected hash.
+2. save to disk the state of which pieces (and partial pieces) are downloaded,
+   and load it back in again when resuming.
+
+If no resume data is provided with a torrent that's added, libtorrent will
+employ (1) by default.
+
+To save resume data, call `save_resume_data()`_ on the torrent_handle_ object.
+This will ask libtorrent to generate the resume data and post it back in
+a save_resume_data_alert_. If generating the resume data fails for any reason,
+a save_resume_data_failed_alert_ is posted instead. Exactly one of those alerts
+will be posted for every call to `save_resume_data()`_.
+
+The save_resume_data_alert_ looks something like this:
+
+.. code:: c++
+
+	struct save_resume_data_alert : torrent_alert
+	{
+		virtual std::string message() const;
+
+		// points to the resume data.
+		boost::shared_ptr<entry> resume_data;
+	};
+
+``resume_data`` points to an entry_ object. This represents a node or a tree
+of nodes in a bencoded_ structure, which is the native encoding scheme in
+bittorrent. It can be encoded into a byte buffer or file using `bencode()`_.
+
+example
+-------
+
+Here's an updated version of the above example with the following updates:
+
+1. not using blocking calls
+2. printing torrent status updates rather than the raw log
+3. saving and loading resume files
+
+.. include:: ../examples/bt-get2.cpp
+	:code: c++
+	:tab-width: 2
+	:start-after: */
 
 .. _session: reference-Core.html#session
 .. _session_handle: reference-Core.html#session_handle
@@ -121,5 +240,22 @@ when a torrent completes.
 .. _`alert`: reference-Alerts.html#alert
 .. _`alert_cast<>`: reference-Alerts.html#alert_cast()
 .. _torrent_finished_alert: reference-Alerts.html#torrent-finished-alert
-
+.. _listen_interfaces: reference-Settings.html#listen_interfaces
+.. _`add_torrent_alert`: reference-Alerts.html#add-torrent-alert
+.. _settings_pack: reference-Settings.html#settings_pack
+.. _alert_mask: reference-Settings.html#alert_mask
+.. _`category flags`: reference-Alerts.html#category_t
+.. _`apply_settings()`: reference-Core.html#apply_settings()
+.. _`session::abort()`: reference-Core.html#abort()
+.. _session_proxy: reference-Core.html#session_proxy
+.. _`post_torrent_updates()`: reference-Core.html#post_torrent_updates()
+.. _torrent_status: reference-Core.html#torrent_status
+.. _state_update_alert: reference-Alerts.html#state_update_alert
+.. _scalability: http://blog.libtorrent.org/2011/11/scalable-interfaces/
+.. _`save_resume_data()`: reference-Core.html#save_resume_data()
+.. _save_resume_data_alert: reference-Alerts.html#save_resume_data_alert
+.. _save_resume_data_failed_alert: reference-Alerts.html#save_resume_data_failed_alert
+.. _bencoded: https://en.wikipedia.org/wiki/Bencode
+.. _entry: reference-Bencoding.html#entry
+.. _`bencode()`: reference-Bencoding.html#bencode()
 

examples/Jamfile

@@ -20,6 +20,8 @@ project client_test
 exe client_test : client_test.cpp print.cpp torrent_view.cpp session_view.cpp ;
 
 exe simple_client : simple_client.cpp ;
+exe bt-get : bt-get.cpp ;
+exe bt-get2 : bt-get2.cpp ;
 exe stats_counters : stats_counters.cpp ;
 exe dump_torrent : dump_torrent.cpp ;
 exe make_torrent : make_torrent.cpp ;
@@ -28,6 +30,8 @@ exe upnp_test : upnp_test.cpp ;
 
 explicit stage_client_test ;
 explicit stage_connection_tester ;
+explicit bt-get ;
+explicit bt-get2 ;
 
 install stage_client_test : client_test : <location>. ;
 install stage_connection_tester : connection_tester : <location>. ;

examples/Makefile.am

@@ -5,6 +5,8 @@ example_programs =  \
   make_torrent      \
   simple_client     \
   upnp_test         \
+  bt_get            \
+  bt_get2           \
   connection_tester
 
 if ENABLE_EXAMPLES
@@ -15,25 +17,14 @@ EXTRA_PROGRAMS = $(example_programs)
 EXTRA_DIST = Jamfile CMakeLists.txt run_cmake.sh.in cmake/FindLibtorrentRasterbar.cmake
 
 client_test_SOURCES = client_test.cpp print.cpp session_view.cpp torrent_view.cpp
-#client_test_LDADD = $(top_builddir)/src/libtorrent-rasterbar.la
-
 stats_counters_SOURCES = stats_counters.cpp
-#stats_counters_LDADD = $(top_builddir)/src/libtorrent-rasterbar.la
+bt_get_SOURCES = bt-get.cpp
-
+bt_get2_SOURCES = bt-get2.cpp
 dump_torrent_SOURCES = dump_torrent.cpp
-#dump_torrent_LDADD = $(top_builddir)/src/libtorrent-rasterbar.la
-
 make_torrent_SOURCES = make_torrent.cpp
-#make_torrent_LDADD = $(top_builddir)/src/libtorrent-rasterbar.la
-
 simple_client_SOURCES = simple_client.cpp
-#simple_client_LDADD = $(top_builddir)/src/libtorrent-rasterbar.la
-
 connection_tester_SOURCES = connection_tester.cpp
-#connection_tester_LDADD = $(top_builddir)/src/libtorrent-rasterbar.la
-
 upnp_test_SOURCES = upnp_test.cpp
-#upnp_test_LDADD = $(top_builddir)/src/libtorrent-rasterbar.la
 
 LDADD = $(top_builddir)/src/libtorrent-rasterbar.la
 

examples/bt-get.cpp

@@ -0,0 +1,75 @@
+/*
+
+Copyright (c) 2016, Arvid Norberg
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+    * Redistributions of source code must retain the above copyright
+      notice, this list of conditions and the following disclaimer.
+    * Redistributions in binary form must reproduce the above copyright
+      notice, this list of conditions and the following disclaimer in
+      the documentation and/or other materials provided with the distribution.
+    * Neither the name of the author nor the names of its
+      contributors may be used to endorse or promote products derived
+      from this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+POSSIBILITY OF SUCH DAMAGE.
+
+*/
+
+#include <iostream>
+#include <thread>
+#include <chrono>
+
+#include <libtorrent/session.hpp>
+#include <libtorrent/add_torrent_params.hpp>
+#include <libtorrent/torrent_handle.hpp>
+#include <libtorrent/alert_types.hpp>
+
+namespace lt = libtorrent;
+int main(int argc, char const* argv[])
+{
+	if (argc != 2) {
+		std::cerr << "usage: " << argv[0] << " <magnet-url>" << std::endl;
+		return 1;
+	}
+	lt::session ses;
+
+	lt::add_torrent_params atp;
+	atp.url = argv[1];
+	atp.save_path = "."; // save in current dir
+	lt::torrent_handle h = ses.add_torrent(atp);
+
+	for (;;) {
+		std::vector<lt::alert*> alerts;
+		ses.pop_alerts(&alerts);
+
+		for (lt::alert const* a : alerts) {
+			std::cout << a->message() << std::endl;
+			// if we receive the finished alert or an error, we're done
+			if (lt::alert_cast<lt::torrent_finished_alert>(a)) {
+				goto done;
+			}
+			if (lt::alert_cast<lt::torrent_error_alert>(a)) {
+				goto done;
+			}
+		}
+		std::this_thread::sleep_for(std::chrono::milliseconds(200));
+	}
+	done:
+	std::cout << "done, shutting down" << std::endl;
+}
+

examples/bt-get2.cpp

@@ -0,0 +1,142 @@
+/*
+
+Copyright (c) 2016, Arvid Norberg
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+    * Redistributions of source code must retain the above copyright
+      notice, this list of conditions and the following disclaimer.
+    * Redistributions in binary form must reproduce the above copyright
+      notice, this list of conditions and the following disclaimer in
+      the documentation and/or other materials provided with the distribution.
+    * Neither the name of the author nor the names of its
+      contributors may be used to endorse or promote products derived
+      from this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+POSSIBILITY OF SUCH DAMAGE.
+
+*/
+
+#include <iostream>
+#include <thread>
+#include <chrono>
+#include <fstream>
+
+#include <libtorrent/session.hpp>
+#include <libtorrent/add_torrent_params.hpp>
+#include <libtorrent/torrent_handle.hpp>
+#include <libtorrent/alert_types.hpp>
+#include <libtorrent/bencode.hpp>
+#include <libtorrent/torrent_status.hpp>
+
+namespace lt = libtorrent;
+
+// return the name of a torrent status enum
+char const* state(lt::torrent_status::state_t s)
+{
+	switch(s)
+	{
+		case lt::torrent_status::checking_files: return "checking";
+		case lt::torrent_status::downloading_metadata: return "dl metadata";
+		case lt::torrent_status::downloading: return "downloading";
+		case lt::torrent_status::finished: return "finished";
+		case lt::torrent_status::seeding: return "seeding";
+		case lt::torrent_status::allocating: return "allocating";
+		case lt::torrent_status::checking_resume_data: return "checking resume";
+		default: return "<>";
+	}
+}
+
+int main(int argc, char const* argv[])
+{
+	if (argc != 2) {
+		std::cerr << "usage: " << argv[0] << " <magnet-url>" << std::endl;
+		return 1;
+	}
+
+	lt::settings_pack pack;
+	pack.set_int(lt::settings_pack::alert_mask
+		, lt::alert::error_notification
+		| lt::alert::storage_notification
+		| lt::alert::status_notification);
+
+	lt::session ses(pack);
+
+	lt::add_torrent_params atp;
+
+	// load resume data from disk and pass it in as we add the magnet link
+	std::ifstream ifs(".resume_file", std::ios_base::binary);
+	ifs.unsetf(std::ios_base::skipws);
+	atp.resume_data.assign(std::istream_iterator<char>(ifs)
+		, std::istream_iterator<char>());
+	atp.url = argv[1];
+	atp.save_path = "."; // save in current dir
+	ses.async_add_torrent(atp);
+
+	// this is the handle we'll set once we get the notification of it being
+	// added
+	lt::torrent_handle h;
+	for (;;) {
+		std::vector<lt::alert*> alerts;
+		ses.pop_alerts(&alerts);
+
+		for (lt::alert const* a : alerts) {
+			if (auto at = lt::alert_cast<lt::add_torrent_alert>(a)) {
+				h = at->handle;
+			}
+			// if we receive the finished alert or an error, we're done
+			if (lt::alert_cast<lt::torrent_finished_alert>(a)) {
+				h.save_resume_data();
+				goto done;
+			}
+			if (lt::alert_cast<lt::torrent_error_alert>(a)) {
+				std::cout << a->message() << std::endl;
+				goto done;
+			}
+
+			// when resume data is ready, save it
+			if (auto rd = lt::alert_cast<lt::save_resume_data_alert>(a)) {
+				std::ofstream of(".resume_file", std::ios_base::binary);
+				of.unsetf(std::ios_base::skipws);
+				lt::bencode(std::ostream_iterator<char>(of)
+					, *rd->resume_data);
+			}
+
+			if (auto st = lt::alert_cast<lt::state_update_alert>(a)) {
+				if (st->status.empty()) continue;
+
+				// we only have a single torrent, so we know which one
+				// the status is for
+				lt::torrent_status const& s = st->status[0];
+				std::cout << "\r" << state(s.state) << " "
+					<< (s.download_payload_rate / 1000) << " kB/s "
+					<< (s.total_done / 1000) << " kB ("
+					<< (s.progress_ppm / 10000) << "%) downloaded\x1b[K";
+				std::cout.flush();
+			}
+		}
+		std::this_thread::sleep_for(std::chrono::milliseconds(200));
+
+		// ask the session to post a state_update_alert, to update our
+		// state output for the torrent
+		ses.post_torrent_updates();
+
+		// TODO: 3 call save_resume_data() once every 30 seconds or so
+	}
+	done:
+	std::cout << "\ndone, shutting down" << std::endl;
+}
+

include/libtorrent/alert_types.hpp

@@ -1942,8 +1942,7 @@ namespace libtorrent
 
 		TORRENT_DEFINE_ALERT(dht_error_alert, 73)
 
-		static const int static_category = alert::error_notification
+		static const int static_category = alert::error_notification | alert::dht_notification;
-			| alert::dht_notification;
 		virtual std::string message() const TORRENT_OVERRIDE;
 
 		// the error code

include/libtorrent/torrent_handle.hpp

@@ -662,9 +662,8 @@ namespace libtorrent
 			only_if_modified = 4
 		};
 
-		// ``save_resume_data()`` generates fast-resume data and returns it as an
+		// ``save_resume_data()`` asks libtorrent to generate fast-resume data for
-		// entry. This entry is suitable for being bencoded. For more information
+		// this torrent.
-		// about how fast-resume works, see fast-resume_.
 		// 
 		// The ``flags`` argument is a bitmask of flags ORed together. see
 		// save_resume_flags_t
@@ -676,9 +675,7 @@ namespace libtorrent
 		// The fast resume data will be empty in the following cases:
 		// 
 		//	1. The torrent handle is invalid.
-		//	2. The torrent is checking (or is queued for checking) its storage, it
+		//	2. The torrent hasn't received valid metadata and was started without
-		//	   will obviously not be ready to write resume data.
-		//	3. The torrent hasn't received valid metadata and was started without
 		//	   metadata (see libtorrent's metadata-from-peers_ extension)
 		// 
 		// Note that by the time you receive the fast resume data, it may already