merged RC_1_1 into master

ChangeLog

@@ -53,6 +53,8 @@
 	* resume data no longer has timestamps of files
 	* require C++11 to build libtorrent
 
+	* fix issue where num_seeds could be greater than num_peers in torrent_status
+	* finished non-seed torrents can also be in super-seeding mode
 	* fix issue related to unloading torrents
 	* fixed finished-time calculation
 	* add missing min_memory_usage() and high_performance_seed() settings presets to python

Makefile.am

@@ -102,7 +102,8 @@ DOCS_PAGES = \
   docs/reference-Settings.html    \
   docs/reference-Storage.html     \
   docs/reference-Utility.html     \
-  docs/reference.html
+  docs/reference.html             \
+  docs/single-page-ref.html
 
 ED25519_SOURCE = \
   ed25519/readme.md \

docs/gen_reference_doc.py

@@ -975,6 +975,8 @@ reference documentation
 
 ''')
 
+out.write('`single-page version`__\n\n__ single-page-ref.html\n\n')
+
 for i in range(4):
 
 	out.write('.. container:: main-toc\n\n')
@@ -993,6 +995,10 @@ for cat in categories:
 :Author: Arvid Norberg, arvid@libtorrent.org
 :Version: 1.2.0
 
+`home`__
+
+__ reference.html
+
 %s
 
 .. contents:: Table of contents

docs/join_rst.py

@@ -0,0 +1,25 @@
+#!/bin/python
+
+import sys
+
+# no processing of the first file
+sys.stdout.write(open(sys.argv[1], 'r').read())
+sys.stderr.write('joining %s\n' % sys.argv[1])
+
+for name in sys.argv[2:]:
+	sys.stdout.write('\n')
+	sys.stderr.write('joining %s\n' % name)
+	f = open(name, 'r')
+	for l in f:
+		# strip out the table of contents from subsequent files
+		if '.. contents::' in l:
+			in_directive = True
+			continue
+		if ':Author:' in l: continue
+		if ':Version:' in l: continue
+
+		if l[0] in ' \t' and in_directive:
+			continue
+		in_directive = False
+		sys.stdout.write(l)
+

docs/makefile

@@ -1,14 +1,16 @@
 # this makefile assumes that you have docutils and rst2pdf installed
+# (python-docutils) as well as aafigure (python-aafigure)
 
 ifneq ($(STAGE),)
 WEB_PATH = ~/Documents/rasterbar/web/products/libtorrent
 endif
 
 ifndef RST2HTML
-RST2HTML:=rst2html-3.4.py
+RST2HTML:=rst2html
 endif
 
 REFERENCE_TARGETS = \
+	manual-ref \
 	reference \
 	reference-Core \
 	reference-Plugins \
@@ -22,8 +24,7 @@ REFERENCE_TARGETS = \
 	reference-Filter \
 	reference-Settings \
 	reference-Bdecoding \
-	reference-ed25519 \
+	reference-ed25519
-	manual-ref
 
 TARGETS = index \
 	udp_tracker_protocol \
@@ -45,6 +46,7 @@ TARGETS = index \
 	hacking \
 	streaming \
 	tutorial \
+	single-page-ref \
 	$(REFERENCE_TARGETS)
 
 FIGURES = \
@@ -66,6 +68,9 @@ epub: $(TARGETS:=.epub) $(FIGURES:=.png)
 
 all: html pdf
 
+single-page-ref.rst: $(REFERENCE_TARGETS:=.rst)
+	python join_rst.py $(filter-out reference.rst, $(REFERENCE_TARGETS:=.rst)) >single-page-ref.rst
+
 settings.rst: ../include/libtorrent/settings_pack.hpp
 	python gen_settings_doc.py
 

docs/template.txt

@@ -21,10 +21,10 @@
 </style>
 %(body_prefix)s
     <div id="container">
-    <table id="header">
+    <a href="index.html"><table id="header">
     <tr><td id="orange"></td>
     <td id="logo">libtorrent</td></tr>
-    </table>
+    </table></a>
     <div id="main">
 %(body_pre_docinfo)s
 %(docinfo)s

docs/template2.txt

@@ -21,10 +21,10 @@
 </style>
 %(body_prefix)s
     <div id="container">
-    <table id="header">
+    <a href="index.html"><table id="header">
     <tr><td id="orange"></td>
     <td id="logo">libtorrent</td></tr>
-    </table>
+    </table></a>
     <div id="main">
 
 <script async src="//pagead2.googlesyndication.com/pagead/js/adsbygoogle.js" type="text/javascript"></script>

include/libtorrent/aux_/file_progress.hpp

@@ -45,6 +45,11 @@ POSSIBILITY OF SUCH DAMAGE.
 #include "libtorrent/invariant_check.hpp"
 #endif
 
+#if TORRENT_USE_INVARIANT_CHECKS
+#include "libtorrent/invariant_check.hpp"
+#include "libtorrent/bitfield.hpp"
+#endif
+
 namespace libtorrent
 {
 class piece_picker;

include/libtorrent/torrent.hpp

@@ -723,7 +723,7 @@ namespace libtorrent
 		bool super_seeding() const
 		{
 			// we're not super seeding if we're not a seed
-			return m_super_seeding && is_seed();
+			return m_super_seeding;
 		}
 
 		void set_super_seeding(bool on);
@@ -1052,12 +1052,26 @@ namespace libtorrent
 			m_links[aux::session_interface::torrent_state_updates].clear();
 		}
 
-		void inc_num_connecting()
+		void inc_num_connecting(torrent_peer* pp)
-		{ ++m_num_connecting; }
+		{
-		void dec_num_connecting()
+			++m_num_connecting;
+			TORRENT_ASSERT(m_num_connecting <= int(m_connections.size()));
+			if (pp->seed)
+			{
+				++m_num_connecting_seeds;
+				TORRENT_ASSERT(m_num_connecting_seeds <= int(m_connections.size()));
+			}
+		}
+		void dec_num_connecting(torrent_peer* pp)
 		{
 			TORRENT_ASSERT(m_num_connecting > 0);
 			--m_num_connecting;
+			if (pp->seed)
+			{
+				TORRENT_ASSERT(m_num_connecting_seeds > 0);
+				--m_num_connecting_seeds;
+			}
+			TORRENT_ASSERT(m_num_connecting <= int(m_connections.size()));
 		}
 
 		bool is_ssl_torrent() const { return m_ssl_torrent; }
@@ -1554,6 +1568,10 @@ namespace libtorrent
 		// counting the peer connections that say true for is_seed()
 		std::uint16_t m_num_seeds = 0;
 
+		// this is the number of peers that are seeds, and count against
+		// m_num_seeds, but have not yet been connected
+		boost::uint16_t m_num_connecting_seeds = 0;
+
 		// the timestamp of the last byte uploaded from this torrent specified in
 		// session_time. This is signed because it must be able to represent time
 		// before the session started.

include/libtorrent/torrent_handle.hpp

@@ -840,6 +840,11 @@ namespace libtorrent
 		void queue_position_top() const;
 		void queue_position_bottom() const;
 
+		// updates the position in the queue for this torrent. The relative order
+		// of all other torrents remain intact but their numerical queue position
+		// shifts to make space for this torrent's new position
+		void queue_position_set(int p) const;
+
 		// For SSL torrents, use this to specify a path to a .pem file to use as
 		// this client's certificate. The certificate must be signed by the
 		// certificate in the .torrent file to be valid.

src/peer_connection.cpp

@@ -153,7 +153,6 @@ namespace libtorrent
 		// if t is nullptr, we better not be connecting, since
 		// we can't decrement the connecting counter
 		TORRENT_ASSERT(t || !m_connecting);
-		if (m_connecting && t) t->inc_num_connecting();
 		m_est_reciprocation_rate = m_settings.get_int(settings_pack::default_est_reciprocation_rate);
 
 		m_channel_state[upload_channel] = peer_info::bw_idle;
@@ -335,6 +334,8 @@ namespace libtorrent
 		// if this is an incoming connection, we're done here
 		if (!m_connecting) return;
 
+		if (m_connecting && t) t->inc_num_connecting(m_peer_info);
+
 #ifndef TORRENT_DISABLE_LOGGING
 		peer_log(peer_log_alert::outgoing, "OPEN", "protocol: %s"
 			, (m_remote.address().is_v4()?"IPv4":"IPv6"));
@@ -778,7 +779,7 @@ namespace libtorrent
 		if (m_connecting)
 		{
 			m_counters.inc_stats_counter(counters::num_peers_half_open, -1);
-			if (t) t->dec_num_connecting();
+			if (t) t->dec_num_connecting(m_peer_info);
 			m_connecting = false;
 		}
 
@@ -3998,7 +3999,7 @@ namespace libtorrent
 		if (m_connecting)
 		{
 			m_counters.inc_stats_counter(counters::num_peers_half_open, -1);
-			if (t) t->dec_num_connecting();
+			if (t) t->dec_num_connecting(m_peer_info);
 			m_connecting = false;
 		}
 
@@ -4214,7 +4215,7 @@ namespace libtorrent
 		if (m_connecting)
 		{
 			m_counters.inc_stats_counter(counters::num_peers_half_open, -1);
-			if (t) t->dec_num_connecting();
+			if (t) t->dec_num_connecting(m_peer_info);
 			m_connecting = false;
 		}
 
@@ -4646,7 +4647,7 @@ namespace libtorrent
 			if (m_connecting)
 			{
 				m_counters.inc_stats_counter(counters::num_peers_half_open, -1);
-				if (t) t->dec_num_connecting();
+				if (t) t->dec_num_connecting(m_peer_info);
 				m_connecting = false;
 			}
 			disconnect(errors::torrent_aborted, op_bittorrent);
@@ -5960,7 +5961,7 @@ namespace libtorrent
 		if (m_connecting)
 		{
 			m_counters.inc_stats_counter(counters::num_peers_half_open, -1);
-			if (t) t->dec_num_connecting();
+			if (t) t->dec_num_connecting(m_peer_info);
 			m_connecting = false;
 		}
 
@@ -5988,7 +5989,7 @@ namespace libtorrent
 		}
 
 		// if there are outgoing interfaces specified, verify this
-		// peer is correctly bound to on of them
+		// peer is correctly bound to one of them
 		if (!m_settings.get_str(settings_pack::outgoing_interfaces).empty())
 		{
 			if (!m_ses.verify_bound_address(m_local.address()

src/session_impl.cpp

@@ -2519,7 +2519,8 @@ namespace aux {
 						i->second->disconnect_peers(1, e);
 					}
 
-					m_settings.set_int(settings_pack::connections_limit, int(m_connections.size()));
+					m_settings.set_int(settings_pack::connections_limit
+						, std::max(10, int(m_connections.size())));
 				}
 				// try again, but still alert the user of the problem
 				async_accept(listener, ssl);
@@ -6288,11 +6289,7 @@ namespace aux {
 	{
 		int limit = m_settings.get_int(settings_pack::connections_limit);
 
-		if (limit <= 0)
+		if (limit <= 0) limit = max_open_files();
-			limit = (std::numeric_limits<int>::max)();
-
-		limit = (std::max)(5, (std::min)(limit
-				, max_open_files() - 20 - m_settings.get_int(settings_pack::file_pool_size)));
 
 		m_settings.set_int(settings_pack::connections_limit, limit);
 

src/torrent.cpp

@@ -670,8 +670,6 @@ namespace libtorrent
 
 		update_gauge();
 
-		m_file_progress.clear();
-
 		update_want_peers();
 		update_want_scrape();
 		update_want_tick();
@@ -2184,11 +2182,11 @@ namespace libtorrent
 			int const blocks_in_last_piece = ((m_torrent_file->total_size() % m_torrent_file->piece_length())
 				+ block_size() - 1) / block_size();
 			m_picker->init(blocks_per_piece, blocks_in_last_piece, m_torrent_file->num_pieces());
+
+			m_file_progress.clear();
+			m_file_progress.init(picker(), m_torrent_file->files());
 		}
 
-		// file progress is allocated lazily, the first time the client
-		// asks for it
-		m_file_progress.clear();
 
 		// assume that we don't have anything
 		m_files_checked = false;
@@ -3187,7 +3185,7 @@ namespace libtorrent
 			return;
 		}
 
-		if (int(m_connections.size()) - m_num_connecting < 10)
+		if (int(m_connections.size() - m_num_connecting) < 10)
 		{
 			// there are too few peers. Be conservative and don't assume it's
 			// well seeded until we can connect to more peers
@@ -3198,8 +3196,8 @@ namespace libtorrent
 		// if there are at least 10 seeds, and there are 10 times more
 		// seeds than downloaders, enter sequential download mode
 		// (for performance)
-		int downloaders = num_downloaders();
+		int const downloaders = num_downloaders();
-		int seeds = num_seeds();
+		int const seeds = num_seeds();
 		m_auto_sequential = downloaders * 10 <= seeds
 			&& seeds > 9;
 	}
@@ -7349,6 +7347,7 @@ namespace libtorrent
 			m_picker.reset();
 			m_have_all = true;
 			update_gauge();
+			m_file_progress.clear();
 		}
 	}
 
@@ -7361,9 +7360,6 @@ namespace libtorrent
 		set_state(torrent_status::seeding);
 		m_became_seed = m_ses.session_time();
 
-		// no need for this anymore
-		m_file_progress.clear();
-
 		if (!m_announcing) return;
 
 		time_point now = aux::time_now();
@@ -7684,6 +7680,9 @@ namespace libtorrent
 #if TORRENT_USE_INVARIANT_CHECKS
 	void torrent::check_invariant() const
 	{
+		// the piece picker and the file progress states are supposed to be
+		// created in sync
+		TORRENT_ASSERT(has_picker() == !m_file_progress.empty());
 		TORRENT_ASSERT(current_stats_state() == int(m_current_gauge_state + counters::num_checking_torrents)
 			|| m_current_gauge_state == no_gauge_state);
 
@@ -7777,6 +7776,8 @@ namespace libtorrent
 
 		int seeds = 0;
 		int num_uploads = 0;
+		int num_connecting = 0;
+		int num_connecting_seeds = 0;
 		std::map<piece_block, int> num_requests;
 		for (const_peer_iterator i = this->begin(); i != this->end(); ++i)
 		{
@@ -7786,6 +7787,11 @@ namespace libtorrent
 #endif
 			peer_connection const& p = *(*i);
 
+			if (p.is_connecting()) ++num_connecting;
+
+			if (p.is_connecting() && p.peer_info_struct()->seed)
+				++num_connecting_seeds;
+
 			if (p.peer_info_struct() && p.peer_info_struct()->seed)
 				++seeds;
 
@@ -7808,6 +7814,14 @@ namespace libtorrent
 		}
 		TORRENT_ASSERT(num_uploads == int(m_num_uploads));
 		TORRENT_ASSERT(seeds == int(m_num_seeds));
+		TORRENT_ASSERT(num_connecting == int(m_num_connecting));
+		TORRENT_ASSERT(num_connecting_seeds == int(m_num_connecting_seeds));
+		TORRENT_ASSERT(int(m_num_uploads) <= int(m_connections.size()));
+		TORRENT_ASSERT(int(m_num_seeds) <= int(m_connections.size()));
+		TORRENT_ASSERT(int(m_num_connecting) <= int(m_connections.size()));
+		TORRENT_ASSERT(int(m_num_connecting_seeds) <= int(m_connections.size()));
+		TORRENT_ASSERT(int(m_num_connecting) + int(m_num_seeds) >= int(m_num_connecting_seeds));
+		TORRENT_ASSERT(int(m_num_connecting) + int(m_num_seeds) - int(m_num_connecting_seeds) <= int(m_connections.size()));
 
 		if (has_picker())
 		{
@@ -7942,6 +7956,9 @@ namespace libtorrent
 
 	void torrent::queue_up()
 	{
+		// fix race conditions on async position change calls (from handler)
+		if(!m_auto_managed || m_abort || is_finished()) return;
+
 		set_queue_position(queue_position() == 0
 			? queue_position() : queue_position() - 1);
 	}
@@ -7954,10 +7971,13 @@ namespace libtorrent
 	void torrent::set_queue_position(int p)
 	{
 		TORRENT_ASSERT(is_single_thread());
+
+		// fix race conditions on async position change calls (from handler)
+		if ((!m_auto_managed || m_abort || is_finished()) && p != -1) return;
+
 		TORRENT_ASSERT((p == -1) == is_finished()
 			|| (!m_auto_managed && p == -1)
 			|| (m_abort && p == -1));
-		if (is_finished() && p != -1) return;
 		if (p == m_sequence_number) return;
 
 		TORRENT_ASSERT(p >= -1);
@@ -10760,20 +10780,27 @@ namespace libtorrent
 		m_stats_counters.inc_stats_counter(counters::recv_failed_bytes, b);
 	}
 
+	// the number of connected peers that are seeds
 	int torrent::num_seeds() const
 	{
 		TORRENT_ASSERT(is_single_thread());
 		INVARIANT_CHECK;
 
-		return m_num_seeds;
+		return int(m_num_seeds) - int(m_num_connecting_seeds);
 	}
 
+	// the number of connected peers that are not seeds
 	int torrent::num_downloaders() const
 	{
 		TORRENT_ASSERT(is_single_thread());
 		INVARIANT_CHECK;
 
-		return (std::max)(0, int(m_connections.size()) - m_num_seeds - m_num_connecting);
+		int const ret = int(m_connections.size())
+			- m_num_seeds
+			- m_num_connecting
+			+ m_num_connecting_seeds;
+		TORRENT_ASSERT(ret >= 0);
+		return ret;
 	}
 
 	void torrent::tracker_request_error(tracker_request const& r

src/torrent_handle.cpp

@@ -355,6 +355,13 @@ namespace libtorrent
 		async_call(&torrent::queue_down);
 	}
 
+	void torrent_handle::queue_position_set(int p) const
+	{
+		TORRENT_ASSERT_PRECOND(p >= 0);
+		if (p < 0) return;
+		TORRENT_ASYNC_CALL1(set_queue_position, p);
+	}
+
 	void torrent_handle::queue_position_top() const
 	{
 		async_call(&torrent::set_queue_position, 0);

test/test_torrent.cpp

@@ -447,3 +447,109 @@ TORRENT_TEST(torrent_status)
 	TEST_EQUAL(static_cast<int>(torrent_status::error_file_exception), -5);
 }
 
+TORRENT_TEST(queue)
+{
+	lt::settings_pack pack = settings();
+	// we're not testing the hash check, just accept the data we write
+	pack.set_bool(settings_pack::disable_hash_checks, true);
+	lt::session ses(pack);
+
+	std::vector<torrent_handle> torrents;
+	for(int i = 0; i < 6; i++)
+	{
+		file_storage fs;
+		std::stringstream file_path;
+		file_path << "test_torrent_dir4/queue" << i;
+		fs.add_file(file_path.str(), 1024);
+		libtorrent::create_torrent t(fs, 128 * 1024, 6);
+
+		std::vector<char> buf;
+		bencode(std::back_inserter(buf), t.generate());
+		std::shared_ptr<torrent_info> ti = std::make_shared<torrent_info>(&buf[0], buf.size());
+		add_torrent_params p;
+		p.ti = ti;
+		p.save_path = ".";
+		torrents.push_back(ses.add_torrent(p));
+	}
+
+	std::vector<int> pieces(0, torrents[5].torrent_file()->num_pieces());
+	torrents[5].prioritize_pieces(pieces);
+	torrent_handle finished = torrents[5];
+
+	// add_torrent should be ordered
+	TEST_EQUAL(finished.queue_position(), -1);
+	TEST_EQUAL(torrents[0].queue_position(), 0);
+	TEST_EQUAL(torrents[1].queue_position(), 1);
+	TEST_EQUAL(torrents[2].queue_position(), 2);
+	TEST_EQUAL(torrents[3].queue_position(), 3);
+	TEST_EQUAL(torrents[4].queue_position(), 4);
+
+	// test top and bottom
+	torrents[2].queue_position_top();
+	torrents[1].queue_position_bottom();
+
+	TEST_EQUAL(finished.queue_position(), -1);
+	TEST_EQUAL(torrents[2].queue_position(), 0);
+	TEST_EQUAL(torrents[0].queue_position(), 1);
+	TEST_EQUAL(torrents[3].queue_position(), 2);
+	TEST_EQUAL(torrents[4].queue_position(), 3);
+	TEST_EQUAL(torrents[1].queue_position(), 4);
+
+	// test set pos
+	torrents[0].queue_position_set(0);
+	torrents[1].queue_position_set(1);
+	// torrent 2 should be get moved down by 0 and 1 to pos 2
+
+	TEST_EQUAL(finished.queue_position(), -1);
+	TEST_EQUAL(torrents[0].queue_position(), 0);
+	TEST_EQUAL(torrents[1].queue_position(), 1);
+	TEST_EQUAL(torrents[2].queue_position(), 2);
+	TEST_EQUAL(torrents[3].queue_position(), 3);
+	TEST_EQUAL(torrents[4].queue_position(), 4);
+
+	//test strange up and down commands
+	torrents[0].queue_position_up();
+	torrents[4].queue_position_down();
+
+	TEST_EQUAL(finished.queue_position(), -1);
+	TEST_EQUAL(torrents[0].queue_position(), 0);
+	TEST_EQUAL(torrents[1].queue_position(), 1);
+	TEST_EQUAL(torrents[2].queue_position(), 2);
+	TEST_EQUAL(torrents[3].queue_position(), 3);
+	TEST_EQUAL(torrents[4].queue_position(), 4);
+
+	torrents[1].queue_position_up();
+	torrents[3].queue_position_down();
+	finished.queue_position_up();
+
+	TEST_EQUAL(finished.queue_position(), -1);
+	TEST_EQUAL(torrents[1].queue_position(), 0);
+	TEST_EQUAL(torrents[0].queue_position(), 1);
+	TEST_EQUAL(torrents[2].queue_position(), 2);
+	TEST_EQUAL(torrents[4].queue_position(), 3);
+	TEST_EQUAL(torrents[3].queue_position(), 4);
+
+	torrents[1].queue_position_down();
+	torrents[3].queue_position_up();
+	finished.queue_position_down();
+
+
+	TEST_EQUAL(finished.queue_position(), -1);
+	TEST_EQUAL(torrents[0].queue_position(), 0);
+	TEST_EQUAL(torrents[1].queue_position(), 1);
+	TEST_EQUAL(torrents[2].queue_position(), 2);
+	TEST_EQUAL(torrents[3].queue_position(), 3);
+	TEST_EQUAL(torrents[4].queue_position(), 4);
+
+	// test set pos on not existing pos
+	torrents[3].queue_position_set(10);
+	finished.queue_position_set(10);
+
+	TEST_EQUAL(finished.queue_position(), -1);
+	TEST_EQUAL(torrents[0].queue_position(), 0);
+	TEST_EQUAL(torrents[1].queue_position(), 1);
+	TEST_EQUAL(torrents[2].queue_position(), 2);
+	TEST_EQUAL(torrents[4].queue_position(), 3);
+	TEST_EQUAL(torrents[3].queue_position(), 4);
+}
+