From c919c63ca3ac9974ab71959d0c192405de2979a8 Mon Sep 17 00:00:00 2001 From: arvidn Date: Sun, 21 Feb 2016 17:09:41 -0500 Subject: [PATCH 1/2] extend the tutorial --- .travis.yml | 3 + appveyor.yml | 3 + docs/tutorial.rst | 206 +++++++++++++++++++++----- examples/Jamfile | 4 + examples/Makefile.am | 17 +-- examples/bt-get.cpp | 75 ++++++++++ examples/bt-get2.cpp | 142 ++++++++++++++++++ include/libtorrent/alert_types.hpp | 3 +- include/libtorrent/torrent_handle.hpp | 9 +- 9 files changed, 406 insertions(+), 56 deletions(-) create mode 100644 examples/bt-get.cpp create mode 100644 examples/bt-get2.cpp diff --git a/.travis.yml b/.travis.yml index 501a85814..65fba8eb1 100644 --- a/.travis.yml +++ b/.travis.yml @@ -91,6 +91,9 @@ script: - cd ../examples - bjam --hash -j3 warnings-as-errors=on variant=$variant $toolset link=shared + - if [[ lang == "cpp11" ]]; then + bjam --hash -j3 warnings-as-errors=on variant=$variant $toolset link=shared bt-get bt-get2; + fi - cd .. # build libtorrent separately and install it in a temporary (well known) dir diff --git a/appveyor.yml b/appveyor.yml index a080522b2..760ed7577 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -117,6 +117,9 @@ build_script: - if not defined x64 ( cd %ROOT_DIRECTORY%\examples & b2.exe --hash -j2 %compiler% variant=%variant% linkflags=%linkflags% include=%include% link=shared + & if %compiler% == msvc-14.0 ( b2.exe --hash -j2 %compiler% variant=%variant% linkflags=%linkflags% include=%include% link=shared bt-get bt-get2 ) + & cd %ROOT_DIRECTORY%\examples + & b2.exe --hash -j2 %compiler% variant=%variant% linkflags=%linkflags% include=%include% link=shared & cd %ROOT_DIRECTORY%\bindings\python & b2.exe --hash -j2 %compiler% stage_module install-dependencies=on variant=%variant% libtorrent-link=shared linkflags=%linkflags% include=%include% & python test.py diff --git a/docs/tutorial.rst b/docs/tutorial.rst index cd165f650..511e52281 100644 --- a/docs/tutorial.rst +++ b/docs/tutorial.rst @@ -29,7 +29,7 @@ For example: #include #include #include - + namespace lt = libtorrent; int main(int argc, char const* argv[]) { @@ -70,46 +70,165 @@ completes downloading, we can poll the session for alerts periodically and print them out, as well as listening for the torrent_finished_alert_, which is posted when a torrent completes. +.. include:: ../examples/bt-get.cpp + :code: c++ + :tab-width: 2 + :start-after: */ + +alert masks +----------- + +The output from this program will be quite verbose, which is probably a good +starting point to get some understanding of what's going on. Alerts are +categorized into alert categories. Each category can be enabled and disabled +independently via the *alert mask*. + +The alert mask is a configuration option offered by libtorrent. There are many +configuration options, see settings_pack_. The alert_mask_ setting is an integer +of the `category flags`_ ORed together. + +For instance, to only see the most pertinent alerts, the session can be +constructed like this: + .. code:: c++ - #include - - #include - #include - #include - #include - - namespace lt = libtorrent; - int main(int argc, char const* argv[]) + lt::settings_pack pack; + pack.set_int(lt::settings_pack::alert_mask + , lt::alert::error_notification + | lt::alert::storage_notification + | lt::alert::status_notification); + + lt::session ses(pack); + +Configuration options can be updated after the session is started by calling +`apply_settings()`_. Some settings are best set before starting the session +though, like listen_interfaces_, to avoid race conditions. If you start the +session with the default settings and then immediately change them, there will +still be a window where the default settings apply. + +Changing the settings may trigger listen sockets to close and re-open and +NAT-PMP, UPnP updates to be sent. For this reason, it's typically a good idea +to batch settings updates into a single call. + +session destruction +------------------- + +The session destructor is blocking by default. When shutting down, trackers +will need to be contacted to stop torrents and other outstanding operations +need to be cancelled. Shutting down can sometimes take several seconds, +primarily because of trackers that are unresponsive (and time out) and also +DNS servers that are unresponsive. DNS lookups are especially difficult to +abort when stalled. + +In order to be able to start destruction an wait for it asynchronously, one +can call `session::abort()`_. + +This call returns a session_proxy_ object, which is a handle keeping the session +state alive while destructing it. It deliberately does not provide any of the +session operations, since it's shutting down. + +After having a session_proxy_ object, the session destructor does not block. +However, the session_proxy_ destructor *will*. + +This can be used to shut down multiple sessions or other parts of the +application in parallel. + +asynchronous operations +----------------------- + +Essentially any call to a member function of session_ or torrent_handle_ that +returns a value is a blocking synchronous call. Meaning it will post a message +to the main libtorrent thread and wait for a response. Such calls may be +expensive, and in applications where stalls should be avoided (such as user +interface threads), blocking calls should be avoided. + +In the example above, session::add_torrent() returns a torrent_handle_ and is +thus blocking. For higher efficiency, `async_add_torrent()`_ will post a message +to the main thread to add a torrent, and post the resulting torrent_handle_ back +in an alert (add_torrent_alert_). This is especially useful when adding a lot +of torrents in quick succession, as there's no stall in between calls. + +In the example above, we don't actually use the torrent_handle_ for anything, so +converting it to use `async_add_torrent()`_ is just a matter of replacing the +`add_torrent()`_ call with `async_add_torrent()`_. + +torrent_status_updates +---------------------- + +To get updates to the status of torrents, call `post_torrent_updates()`_ on the +session object. This will cause libtorrent to post a state_update_alert_ +containing torrent_status_ objects for all torrents whose status has *changed* +since the last call to `post_torrent_updates()`_. + +The state_update_alert_ looks something like this: + +.. code:: c++ + + struct state_update_alert : alert { - if (argc != 2) { - std::cerr << "usage: " << argv[0] << " " << std::endl; - return 1; - } - lt::session ses; + virtual std::string message() const; + std::vector status; + }; - lt::add_torrent_params atp; - atp.url = argv[1]; - atp.save_path = "."; // save in current dir - lt::torrent_handle h = ses.add_torrent(atp); +The ``status`` field only contains the torrent_status_ for torrents with +updates since the last call. It may be empty if no torrent has updated its +state. This feature is critical for scalability_. - bool done = false; - while (!done) { - std::vector alerts; - ses.pop_alerts(&alerts); +See the torrent_status_ object for more information on what is in there. +Perhaps the most interesting fields are ``total_payload_download``, +``total_payload_upload``, ``num_peers`` and ``state``. - for (lt::alert const* a : alerts) { - std::cout << a->message() << std::endl; - if (lt::alert_cast(a)) { - done = true; - } - } - } - } +resuming torrents +----------------- -*TODO* cover async_add_torrent() -*TODO* cover post_torrent_updates() -*TODO* cover save_resume_data() +Since bittorrent downloads pieces of files in random order, it's not trivial to +resume a partial download. When resuming a download, the bittorrent engine must +restore the state of the downloading torrent, specifically, which parts of the +file(s) are downloaded. There are two approaches to doing this: + +1. read every piece of the downloaded files from disk and compare it against the + expected hash. +2. save to disk the state of which pieces (and partial pieces) are downloaded, + and load it back in again when resuming. + +If no resume data is provided with a torrent that's added, libtorrent will +employ (1) by default. + +To save resume data, call `save_resume_data()`_ on the torrent_handle_ object. +This will ask libtorrent to generate the resume data and post it back in +a save_resume_data_alert_. If generating the resume data fails for any reason, +a save_resume_data_failed_alert_ is posted instead. Exactly one of those alerts +will be posted for every call to `save_resume_data()`_. + +The save_resume_data_alert_ looks something like this: + +.. code:: c++ + + struct save_resume_data_alert : torrent_alert + { + virtual std::string message() const; + + // points to the resume data. + boost::shared_ptr resume_data; + }; + +``resume_data`` points to an entry_ object. This represents a node or a tree +of nodes in a bencoded_ structure, which is the native encoding scheme in +bittorrent. It can be encoded into a byte buffer or file using `bencode()`_. + +example +------- + +Here's an updated version of the above example with the following updates: + +1. not using blocking calls +2. printing torrent status updates rather than the raw log +3. saving and loading resume files + +.. include:: ../examples/bt-get2.cpp + :code: c++ + :tab-width: 2 + :start-after: */ .. _session: reference-Core.html#session .. _session_handle: reference-Core.html#session_handle @@ -121,5 +240,22 @@ when a torrent completes. .. _`alert`: reference-Alerts.html#alert .. _`alert_cast<>`: reference-Alerts.html#alert_cast() .. _torrent_finished_alert: reference-Alerts.html#torrent-finished-alert - +.. _listen_interfaces: reference-Settings.html#listen_interfaces +.. _`add_torrent_alert`: reference-Alerts.html#add-torrent-alert +.. _settings_pack: reference-Settings.html#settings_pack +.. _alert_mask: reference-Settings.html#alert_mask +.. _`category flags`: reference-Alerts.html#category_t +.. _`apply_settings()`: reference-Core.html#apply_settings() +.. _`session::abort()`: reference-Core.html#abort() +.. _session_proxy: reference-Core.html#session_proxy +.. _`post_torrent_updates()`: reference-Core.html#post_torrent_updates() +.. _torrent_status: reference-Core.html#torrent_status +.. _state_update_alert: reference-Alerts.html#state_update_alert +.. _scalability: http://blog.libtorrent.org/2011/11/scalable-interfaces/ +.. _`save_resume_data()`: reference-Core.html#save_resume_data() +.. _save_resume_data_alert: reference-Alerts.html#save_resume_data_alert +.. _save_resume_data_failed_alert: reference-Alerts.html#save_resume_data_failed_alert +.. _bencoded: https://en.wikipedia.org/wiki/Bencode +.. _entry: reference-Bencoding.html#entry +.. _`bencode()`: reference-Bencoding.html#bencode() diff --git a/examples/Jamfile b/examples/Jamfile index 0b6057d47..6c885ac1a 100644 --- a/examples/Jamfile +++ b/examples/Jamfile @@ -20,6 +20,8 @@ project client_test exe client_test : client_test.cpp print.cpp torrent_view.cpp session_view.cpp ; exe simple_client : simple_client.cpp ; +exe bt-get : bt-get.cpp ; +exe bt-get2 : bt-get2.cpp ; exe stats_counters : stats_counters.cpp ; exe dump_torrent : dump_torrent.cpp ; exe make_torrent : make_torrent.cpp ; @@ -28,6 +30,8 @@ exe upnp_test : upnp_test.cpp ; explicit stage_client_test ; explicit stage_connection_tester ; +explicit bt-get ; +explicit bt-get2 ; install stage_client_test : client_test : . ; install stage_connection_tester : connection_tester : . ; diff --git a/examples/Makefile.am b/examples/Makefile.am index ccedaa87b..4ced3561c 100644 --- a/examples/Makefile.am +++ b/examples/Makefile.am @@ -5,6 +5,8 @@ example_programs = \ make_torrent \ simple_client \ upnp_test \ + bt_get \ + bt_get2 \ connection_tester if ENABLE_EXAMPLES @@ -15,25 +17,14 @@ EXTRA_PROGRAMS = $(example_programs) EXTRA_DIST = Jamfile CMakeLists.txt run_cmake.sh.in cmake/FindLibtorrentRasterbar.cmake client_test_SOURCES = client_test.cpp print.cpp session_view.cpp torrent_view.cpp -#client_test_LDADD = $(top_builddir)/src/libtorrent-rasterbar.la - stats_counters_SOURCES = stats_counters.cpp -#stats_counters_LDADD = $(top_builddir)/src/libtorrent-rasterbar.la - +bt_get_SOURCES = bt-get.cpp +bt_get2_SOURCES = bt-get2.cpp dump_torrent_SOURCES = dump_torrent.cpp -#dump_torrent_LDADD = $(top_builddir)/src/libtorrent-rasterbar.la - make_torrent_SOURCES = make_torrent.cpp -#make_torrent_LDADD = $(top_builddir)/src/libtorrent-rasterbar.la - simple_client_SOURCES = simple_client.cpp -#simple_client_LDADD = $(top_builddir)/src/libtorrent-rasterbar.la - connection_tester_SOURCES = connection_tester.cpp -#connection_tester_LDADD = $(top_builddir)/src/libtorrent-rasterbar.la - upnp_test_SOURCES = upnp_test.cpp -#upnp_test_LDADD = $(top_builddir)/src/libtorrent-rasterbar.la LDADD = $(top_builddir)/src/libtorrent-rasterbar.la diff --git a/examples/bt-get.cpp b/examples/bt-get.cpp new file mode 100644 index 000000000..0029a81bf --- /dev/null +++ b/examples/bt-get.cpp @@ -0,0 +1,75 @@ +/* + +Copyright (c) 2016, Arvid Norberg +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the distribution. + * Neither the name of the author nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. + +*/ + +#include +#include +#include + +#include +#include +#include +#include + +namespace lt = libtorrent; +int main(int argc, char const* argv[]) +{ + if (argc != 2) { + std::cerr << "usage: " << argv[0] << " " << std::endl; + return 1; + } + lt::session ses; + + lt::add_torrent_params atp; + atp.url = argv[1]; + atp.save_path = "."; // save in current dir + lt::torrent_handle h = ses.add_torrent(atp); + + for (;;) { + std::vector alerts; + ses.pop_alerts(&alerts); + + for (lt::alert const* a : alerts) { + std::cout << a->message() << std::endl; + // if we receive the finished alert or an error, we're done + if (lt::alert_cast(a)) { + goto done; + } + if (lt::alert_cast(a)) { + goto done; + } + } + std::this_thread::sleep_for(std::chrono::milliseconds(200)); + } + done: + std::cout << "done, shutting down" << std::endl; +} + diff --git a/examples/bt-get2.cpp b/examples/bt-get2.cpp new file mode 100644 index 000000000..ab0873747 --- /dev/null +++ b/examples/bt-get2.cpp @@ -0,0 +1,142 @@ +/* + +Copyright (c) 2016, Arvid Norberg +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the distribution. + * Neither the name of the author nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. + +*/ + +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +namespace lt = libtorrent; + +// return the name of a torrent status enum +char const* state(lt::torrent_status::state_t s) +{ + switch(s) + { + case lt::torrent_status::checking_files: return "checking"; + case lt::torrent_status::downloading_metadata: return "dl metadata"; + case lt::torrent_status::downloading: return "downloading"; + case lt::torrent_status::finished: return "finished"; + case lt::torrent_status::seeding: return "seeding"; + case lt::torrent_status::allocating: return "allocating"; + case lt::torrent_status::checking_resume_data: return "checking resume"; + default: return "<>"; + } +} + +int main(int argc, char const* argv[]) +{ + if (argc != 2) { + std::cerr << "usage: " << argv[0] << " " << std::endl; + return 1; + } + + lt::settings_pack pack; + pack.set_int(lt::settings_pack::alert_mask + , lt::alert::error_notification + | lt::alert::storage_notification + | lt::alert::status_notification); + + lt::session ses(pack); + + lt::add_torrent_params atp; + + // load resume data from disk and pass it in as we add the magnet link + std::ifstream ifs(".resume_file", std::ios_base::binary); + ifs.unsetf(std::ios_base::skipws); + atp.resume_data.assign(std::istream_iterator(ifs) + , std::istream_iterator()); + atp.url = argv[1]; + atp.save_path = "."; // save in current dir + ses.async_add_torrent(atp); + + // this is the handle we'll set once we get the notification of it being + // added + lt::torrent_handle h; + for (;;) { + std::vector alerts; + ses.pop_alerts(&alerts); + + for (lt::alert const* a : alerts) { + if (auto at = lt::alert_cast(a)) { + h = at->handle; + } + // if we receive the finished alert or an error, we're done + if (lt::alert_cast(a)) { + h.save_resume_data(); + goto done; + } + if (lt::alert_cast(a)) { + std::cout << a->message() << std::endl; + goto done; + } + + // when resume data is ready, save it + if (auto rd = lt::alert_cast(a)) { + std::ofstream of(".resume_file", std::ios_base::binary); + of.unsetf(std::ios_base::skipws); + lt::bencode(std::ostream_iterator(of) + , *rd->resume_data); + } + + if (auto st = lt::alert_cast(a)) { + if (st->status.empty()) continue; + + // we only have a single torrent, so we know which one + // the status is for + lt::torrent_status const& s = st->status[0]; + std::cout << "\r" << state(s.state) << " " + << (s.download_payload_rate / 1000) << " kB/s " + << (s.total_done / 1000) << " kB (" + << (s.progress_ppm / 10000) << "%) downloaded\x1b[K"; + std::cout.flush(); + } + } + std::this_thread::sleep_for(std::chrono::milliseconds(200)); + + // ask the session to post a state_update_alert, to update our + // state output for the torrent + ses.post_torrent_updates(); + + // TODO: 3 call save_resume_data() once every 30 seconds or so + } + done: + std::cout << "\ndone, shutting down" << std::endl; +} + diff --git a/include/libtorrent/alert_types.hpp b/include/libtorrent/alert_types.hpp index 6b995ddba..da9ce25bf 100644 --- a/include/libtorrent/alert_types.hpp +++ b/include/libtorrent/alert_types.hpp @@ -1942,8 +1942,7 @@ namespace libtorrent TORRENT_DEFINE_ALERT(dht_error_alert, 73) - static const int static_category = alert::error_notification - | alert::dht_notification; + static const int static_category = alert::error_notification | alert::dht_notification; virtual std::string message() const TORRENT_OVERRIDE; // the error code diff --git a/include/libtorrent/torrent_handle.hpp b/include/libtorrent/torrent_handle.hpp index c04dfaadb..dcb528467 100644 --- a/include/libtorrent/torrent_handle.hpp +++ b/include/libtorrent/torrent_handle.hpp @@ -662,9 +662,8 @@ namespace libtorrent only_if_modified = 4 }; - // ``save_resume_data()`` generates fast-resume data and returns it as an - // entry. This entry is suitable for being bencoded. For more information - // about how fast-resume works, see fast-resume_. + // ``save_resume_data()`` asks libtorrent to generate fast-resume data for + // this torrent. // // The ``flags`` argument is a bitmask of flags ORed together. see // save_resume_flags_t @@ -676,9 +675,7 @@ namespace libtorrent // The fast resume data will be empty in the following cases: // // 1. The torrent handle is invalid. - // 2. The torrent is checking (or is queued for checking) its storage, it - // will obviously not be ready to write resume data. - // 3. The torrent hasn't received valid metadata and was started without + // 2. The torrent hasn't received valid metadata and was started without // metadata (see libtorrent's metadata-from-peers_ extension) // // Note that by the time you receive the fast resume data, it may already From 5d7fbd42c786889bb403b82c952de1f949fe7203 Mon Sep 17 00:00:00 2001 From: arvidn Date: Sun, 21 Feb 2016 23:17:47 -0500 Subject: [PATCH 2/2] clean up TORRENT_FINAL and TORRENT_OVERRIDE in documentation --- docs/gen_reference_doc.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/gen_reference_doc.py b/docs/gen_reference_doc.py index 10c0645f4..029e95801 100644 --- a/docs/gen_reference_doc.py +++ b/docs/gen_reference_doc.py @@ -149,6 +149,7 @@ def is_visible(desc): return True def highlight_signature(s): + s = s.replace('TORRENT_OVERRIDE', 'override').replace('TORRENT_FINAL', 'final') name = s.split('(', 1) name2 = name[0].split(' ') if len(name2[-1]) == 0: return s @@ -273,7 +274,8 @@ def parse_class(lno, lines, filename): state = 'private' class_type = 'class' - name = decl.split(':')[0].replace('class ', '').replace('struct ', '').replace('TORRENT_FINAL', '').strip() + decl = decl.replace('TORRENT_FINAL', 'final') + name = decl.split(':')[0].replace('class ', '').replace('struct ', '').replace('final', '').strip() while lno < len(lines):