added prioritization in the bandwidth_manager (throttler), web seeds now have lower priority when being handed bandwidth. torrent_reply_alert now includes the number of peers returned from the tracker. updated Changelog

ChangeLog

@@ -1,10 +1,14 @@
-	* Added the possibility to have libtorrent resolve the countries of
+	* fixed a bug in the DHT which could be triggered if the network was
+	  dropped or extremely rare cases.
+	* if the download rate is limited, web seeds will now only use left-over
+	  bandwidth after all bt peers have used up as much bandwidth as they can.
+	* added the possibility to have libtorrent resolve the countries of
 	  the peers in torrents.
-	* Improved the bandwidth limiter (it now implements a leaky bucket).
-	* Improved the HTTP seed downloader to report accurate progress.
-	* Added more client peer-id signatures to be recognized.
+	* improved the bandwidth limiter (it now implements a leaky bucket/node bucket).
+	* improved the HTTP seed downloader to report accurate progress.
+	* added more client peer-id signatures to be recognized.
 	* added support for HTTP servers that skip the CR before the NL at line breaks.
-	* fixed bug in the HTTP code that onlcy accepted headers case sensitive.
+	* fixed bug in the HTTP code that only accepted headers case sensitive.
 	* fixed bug where one of the session constructors didn't initialize boost.filesystem. 
 	* fixed bug when the initial checking of a torrent fails with an exception.
 	* fixed bug in DHT code which would send incorrect announce messages.

docs/manual.html

@@ -2505,16 +2505,22 @@ struct tracker_alert: torrent_alert
 <div class="section">
 <h2><a id="tracker-reply-alert" name="tracker-reply-alert">tracker_reply_alert</a></h2>
 <p>This alert is only for informational purpose. It is generated when a tracker announce
-succeeds. It is generated with severity level <tt class="docutils literal"><span class="pre">info</span></tt>.</p>
+succeeds. It is generated regardless what kind of tracker was used, be it UDP, HTTP or
+the DHT. It is generated with severity level <tt class="docutils literal"><span class="pre">info</span></tt>.</p>
 <pre class="literal-block">
 struct tracker_reply_alert: torrent_alert
 {
         tracker_reply_alert(const torrent_handle&amp; h
+                , int num_peers
                 , const std::string&amp; msg);
 
+        int num_peers;
+
         virtual std::auto_ptr&lt;alert&gt; clone() const;
 };
 </pre>
+<p>The <tt class="docutils literal"><span class="pre">num_peers</span></tt> tells how many peers were returned from the tracker. This is
+not necessarily all new peers, some of them may already be connected.</p>
 </div>
 <div class="section">
 <h2><a id="tracker-warning-alert" name="tracker-warning-alert">tracker_warning_alert</a></h2>

docs/manual.rst

@@ -2558,17 +2558,24 @@ tracker_reply_alert
 -------------------
 
 This alert is only for informational purpose. It is generated when a tracker announce
-succeeds. It is generated with severity level ``info``.
+succeeds. It is generated regardless what kind of tracker was used, be it UDP, HTTP or
+the DHT. It is generated with severity level ``info``.
 
 ::
 
 	struct tracker_reply_alert: torrent_alert
 	{
 		tracker_reply_alert(const torrent_handle& h
+			, int num_peers
 			, const std::string& msg);
 
+		int num_peers;
+
 		virtual std::auto_ptr<alert> clone() const;
 	};
+
+The ``num_peers`` tells how many peers were returned from the tracker. This is
+not necessarily all new peers, some of them may already be connected.
 	
 tracker_warning_alert
 ---------------------

examples/client_test.cpp

@@ -806,6 +806,11 @@ int main(int ac, char* av[])
 				{
 					events.push_back(now + ": tracker message: " + p->msg());
 				}
+				else if (tracker_reply_alert* p = dynamic_cast<tracker_reply_alert*>(a.get()))
+				{
+					events.push_back(now + ": " + p->msg() + " ("
+						+ boost::lexical_cast<std::string>(p->num_peers) + ")");
+				}
 				else
 				{
 					events.push_back(now + ": " + a->msg());

include/libtorrent/alert_types.hpp

@@ -86,10 +86,14 @@ namespace libtorrent
 	struct TORRENT_EXPORT tracker_reply_alert: torrent_alert
 	{
 		tracker_reply_alert(torrent_handle const& h
+			, int np
 			, std::string const& msg)
 			: torrent_alert(h, alert::info, msg)
+			, num_peers(np)
 		{}
 
+		int num_peers;
+
 		virtual std::auto_ptr<alert> clone() const
 		{ return std::auto_ptr<alert>(new tracker_reply_alert(*this)); }
 	};

include/libtorrent/bandwidth_manager.hpp

@@ -77,6 +77,13 @@ struct history_entry
 	weak_ptr<torrent> tor;
 };
 
+struct bw_queue_entry
+{
+	bw_queue_entry(boost::intrusive_ptr<peer_connection> const& pe, bool no_prio);
+	boost::intrusive_ptr<peer_connection> peer;
+	bool non_prioritized;
+};
+
 // member of peer_connection
 struct bandwidth_limit
 {
@@ -169,7 +176,11 @@ struct bandwidth_manager
 		return m_limit;
 	}
 
-	void request_bandwidth(intrusive_ptr<peer_connection> peer);
+	// non prioritized means that, if there's a line for bandwidth,
+	// others will cut in front of the non-prioritized peers.
+	// this is used by web seeds
+	void request_bandwidth(intrusive_ptr<peer_connection> peer
+		, bool non_prioritized);
 
 #ifndef NDEBUG
 	void check_invariant() const;
@@ -202,7 +213,7 @@ private:
 	int m_current_quota;
 
 	// these are the consumers that want bandwidth
-	std::deque<intrusive_ptr<peer_connection> > m_queue;
+	std::deque<bw_queue_entry> m_queue;
 
 	// these are the consumers that have received bandwidth
 	// that will expire

include/libtorrent/peer_connection.hpp

@@ -156,6 +156,9 @@ namespace libtorrent
 		void request_large_blocks(bool b)
 		{ m_request_large_blocks = b; }
 
+		void set_non_prioritized(bool b)
+		{ m_non_prioritized = b; }
+
 		// this adds an announcement in the announcement queue
 		// it will let the peer know that we have the given piece
 		void announce_piece(int index);
@@ -611,6 +614,12 @@ namespace libtorrent
 		// the http-downloader, to request whole pieces
 		// at a time.
 		bool m_request_large_blocks;
+		
+		// if this is true, other (prioritized) peers will
+		// skip ahead of it in the queue for bandwidth. The
+		// effect is that non prioritized peers will only use
+		// the left-over bandwidth (suitable for web seeds).
+		bool m_non_prioritized;
 
 		// reference counter for intrusive_ptr
 		mutable boost::detail::atomic_count m_refs;

include/libtorrent/torrent.hpp

@@ -201,7 +201,8 @@ namespace libtorrent
 		bandwidth_limit m_bandwidth_limit[2];
 
 		void request_bandwidth(int channel
-			, boost::intrusive_ptr<peer_connection> p);
+			, boost::intrusive_ptr<peer_connection> p
+			, bool non_prioritized);
 		
 		void expire_bandwidth(int channel, int amount);
 		void assign_bandwidth(int channel, int amount);
@@ -564,7 +565,7 @@ namespace libtorrent
 		boost::scoped_ptr<piece_picker> m_picker;
 
 		// the queue of peer_connections that want more bandwidth
-		std::deque<intrusive_ptr<peer_connection> > m_bandwidth_queue[2];
+		std::deque<bw_queue_entry> m_bandwidth_queue[2];
 
 		std::vector<announce_entry> m_trackers;
 		// this is an index into m_torrent_file.trackers()

src/bandwidth_manager.cpp

@@ -48,6 +48,11 @@ namespace libtorrent
 		, weak_ptr<torrent> t, int a, pt::ptime exp)
 		: expires_at(exp), amount(a), peer(p), tor(t)
 	{}
+	
+	bw_queue_entry::bw_queue_entry(intrusive_ptr<peer_connection> const& pe
+		, bool no_prio)
+		: peer(pe), non_prioritized(no_prio)
+	{}
 
 	bandwidth_manager::bandwidth_manager(io_service& ios, int channel)
 		: m_ios(ios)
@@ -57,23 +62,39 @@ namespace libtorrent
 		, m_channel(channel)
 	{}
 
-	void bandwidth_manager::request_bandwidth(intrusive_ptr<peer_connection> peer)
+	void bandwidth_manager::request_bandwidth(intrusive_ptr<peer_connection> peer
+		, bool non_prioritized)
 	{
 		INVARIANT_CHECK;
 
 		// make sure this peer isn't already in line
 		// waiting for bandwidth
 #ifndef NDEBUG
-		for (std::deque<intrusive_ptr<peer_connection> >::iterator i = m_queue.begin()
+		for (std::deque<bw_queue_entry>::iterator i = m_queue.begin()
 			, end(m_queue.end()); i != end; ++i)
 		{
-			assert(*i < peer || peer < *i);
+			assert(i->peer < peer || peer < i->peer);
 		}
 #endif
 
 		assert(peer->max_assignable_bandwidth(m_channel) > 0);
-
-		m_queue.push_back(peer);
+		
+		// if the queue is empty, we have to push the new
+		// peer at the back of it. If the peer is non-prioritized
+		// it is not supposed to cut in fron of anybody, so then
+		// we also just add it at the end
+		if (m_queue.empty() || non_prioritized)
+		{
+			m_queue.push_back(bw_queue_entry(peer, non_prioritized));
+		}
+		else
+		{
+			// skip forward in the queue until we find a prioritized peer
+			// or hit the front of it.
+			std::deque<bw_queue_entry>::reverse_iterator i = m_queue.rbegin();
+			while (i != m_queue.rend() && i->non_prioritized) ++i;
+			m_queue.insert(i.base(), bw_queue_entry(peer, non_prioritized));
+		}
 		if (m_queue.size() == 1) hand_out_bandwidth();
 	}
 
@@ -176,12 +197,12 @@ namespace libtorrent
 		while (!m_queue.empty() && amount > 0)
 		{
 			assert(amount == limit - m_current_quota);
-			intrusive_ptr<peer_connection> peer = m_queue.front();
+			bw_queue_entry qe = m_queue.front();
 			m_queue.pop_front();
 
-			shared_ptr<torrent> t = peer->associated_torrent().lock();
+			shared_ptr<torrent> t = qe.peer->associated_torrent().lock();
 			if (!t) continue;
-			if (peer->is_disconnecting())
+			if (qe.peer->is_disconnecting())
 			{
 				t->expire_bandwidth(m_channel, -1);
 				continue;
@@ -191,7 +212,7 @@ namespace libtorrent
 			// the bandwidth quota is subtracted once the data has been
 			// send. If the peer was added to the queue while the data was
 			// still being sent, max_assignable may have been > 0 at that time.
-			int max_assignable = peer->max_assignable_bandwidth(m_channel);
+			int max_assignable = qe.peer->max_assignable_bandwidth(m_channel);
 			if (max_assignable == 0)
 			{
 				t->expire_bandwidth(m_channel, -1);
@@ -206,9 +227,9 @@ namespace libtorrent
 					, max_assignable));
 			assert(single_amount > 0);
 			amount -= single_amount;
-			peer->assign_bandwidth(m_channel, single_amount);
+			qe.peer->assign_bandwidth(m_channel, single_amount);
 			t->assign_bandwidth(m_channel, single_amount);
-			add_history_entry(history_entry(peer, t, single_amount, now + window_size));
+			add_history_entry(history_entry(qe.peer, t, single_amount, now + window_size));
 		}
 	}
 	catch (std::exception& e)

src/http_tracker_connection.cpp

@@ -641,6 +641,13 @@ namespace libtorrent
 				fail(-1, error_str.c_str());
 				return;
 			}
+			
+			// if the protocol isn't specified, assume http
+			if (location.compare(0, 7, "http://") != 0
+				&& location.compare(0, 6, "udp://") != 0)
+			{
+				location.insert(0, "http://");
+			}
 
 #if defined(TORRENT_VERBOSE_LOGGING) || defined(TORRENT_LOGGING)
 			if (has_requester()) requester().debug_log("Redirecting to \"" + location + "\"");

src/peer_connection.cpp

@@ -116,6 +116,7 @@ namespace libtorrent
 		, m_reading(false)
 		, m_prefer_whole_pieces(false)
 		, m_request_large_blocks(false)
+		, m_non_prioritized(false)
 		, m_refs(0)
 		, m_upload_limit(resource_request::inf)
 		, m_download_limit(resource_request::inf)
@@ -179,6 +180,7 @@ namespace libtorrent
 		, m_reading(false)
 		, m_prefer_whole_pieces(false)
 		, m_request_large_blocks(false)
+		, m_non_prioritized(false)
 		, m_refs(0)
 		, m_upload_limit(resource_request::inf)
 		, m_download_limit(resource_request::inf)
@@ -1760,7 +1762,7 @@ namespace libtorrent
 				(*m_logger) << "req bandwidth [ " << upload_channel << " ]\n";
 #endif
 
-				t->request_bandwidth(upload_channel, self());
+				t->request_bandwidth(upload_channel, self(), m_non_prioritized);
 				m_writing = true;
 			}
 			return;
@@ -1817,7 +1819,7 @@ namespace libtorrent
 #ifdef TORRENT_VERBOSE_LOGGING
 				(*m_logger) << "req bandwidth [ " << download_channel << " ]\n";
 #endif
-				t->request_bandwidth(download_channel, self());
+				t->request_bandwidth(download_channel, self(), m_non_prioritized);
 				m_reading = true;
 			}
 			return;

src/torrent.cpp

@@ -479,6 +479,13 @@ namespace libtorrent
 
 	void torrent::on_dht_announce_response(std::vector<tcp::endpoint> const& peers)
 	{
+		if (peers.empty()) return;
+
+		if (m_ses.m_alerts.should_post(alert::info))
+		{
+			m_ses.m_alerts.post_alert(tracker_reply_alert(
+				get_handle(), peers.size(), "Got peers from DHT"));
+		}
 		std::for_each(peers.begin(), peers.end(), bind(
 			&policy::peer_from_tracker, boost::ref(m_policy), _1, peer_id(0)));
 	}
@@ -583,7 +590,7 @@ namespace libtorrent
 			s << "Got response from tracker: "
 				<< m_trackers[m_last_working_tracker].url;
 			m_ses.m_alerts.post_alert(tracker_reply_alert(
-				get_handle(), s.str()));
+				get_handle(), peer_list.size(), s.str()));
 		}
 		m_got_tracker_response = true;
 	}
@@ -1773,19 +1780,20 @@ namespace libtorrent
 	}
 
 	void torrent::request_bandwidth(int channel
-		, boost::intrusive_ptr<peer_connection> p)
+		, boost::intrusive_ptr<peer_connection> p
+		, bool non_prioritized)
 	{
 		if (m_bandwidth_limit[channel].max_assignable() >= max_bandwidth_block_size)
 		{
 			if (channel == peer_connection::upload_channel)
-				m_ses.m_ul_bandwidth_manager.request_bandwidth(p);
+				m_ses.m_ul_bandwidth_manager.request_bandwidth(p, non_prioritized);
 			else if (channel == peer_connection::download_channel)
-				m_ses.m_dl_bandwidth_manager.request_bandwidth(p);
+				m_ses.m_dl_bandwidth_manager.request_bandwidth(p, non_prioritized);
 			m_bandwidth_limit[channel].assign(max_bandwidth_block_size);
 		}
 		else
 		{
-			m_bandwidth_queue[channel].push_back(p);
+			m_bandwidth_queue[channel].push_back(bw_queue_entry(p, non_prioritized));
 		}
 	}
 
@@ -1800,12 +1808,12 @@ namespace libtorrent
 		while (!m_bandwidth_queue[channel].empty()
 			&& m_bandwidth_limit[channel].max_assignable() >= max_bandwidth_block_size)
 		{
-			intrusive_ptr<peer_connection> p = m_bandwidth_queue[channel].front();
+			bw_queue_entry qe = m_bandwidth_queue[channel].front();
 			m_bandwidth_queue[channel].pop_front();
 			if (channel == peer_connection::upload_channel)
-				m_ses.m_ul_bandwidth_manager.request_bandwidth(p);
+				m_ses.m_ul_bandwidth_manager.request_bandwidth(qe.peer, qe.non_prioritized);
 			else if (channel == peer_connection::download_channel)
-				m_ses.m_dl_bandwidth_manager.request_bandwidth(p);
+				m_ses.m_dl_bandwidth_manager.request_bandwidth(qe.peer, qe.non_prioritized);
 			m_bandwidth_limit[channel].assign(max_bandwidth_block_size);
 		}
 	}

src/web_peer_connection.cpp

@@ -70,7 +70,11 @@ namespace libtorrent
 		// we always prefer downloading entire
 		// pieces from web seeds
 		prefer_whole_pieces(true);
+		// we want large blocks as well, so
+		// we can request more bytes at once
 		request_large_blocks(true);
+		// we only want left-over bandwidth
+		set_non_prioritized(true);
 		shared_ptr<torrent> tor = t.lock();
 		assert(tor);
 		int blocks_per_piece = tor->torrent_file().piece_length() / tor->block_size();