added logging of upload and download statistics, revised some of the quota management, added support for sessions to allocate_resources (patch by wojci)

include/libtorrent/allocate_resources.hpp

@@ -41,6 +41,7 @@ POSSIBILITY OF SUCH DAMAGE.
 #include "libtorrent/resource_request.hpp"
 #include "libtorrent/peer_id.hpp"
 #include "libtorrent/socket.hpp"
+#include "libtorrent/session.hpp"
 
 namespace libtorrent
 {
@@ -66,6 +67,11 @@ namespace libtorrent
 		, std::map<tcp::endpoint, peer_connection*>& connections
 		, resource_request peer_connection::* res);
 
+	// Used for global limits.
+	void allocate_resources(
+		int resources
+		, std::vector<session*>& _sessions
+		, resource_request session::* res);
 }
 
 

include/libtorrent/aux_/allocate_resources_impl.hpp

@@ -151,6 +151,12 @@ namespace libtorrent
 				, res);
 	#endif
 
+			for (It i = start; i != end; ++i)
+			{
+				resource_request& r = (*i).*res;
+				r.leftovers = (std::max)(r.used - r.given, 0);
+			}
+
 			if (resources == resource_request::inf)
 			{
 				// No competition for resources.
@@ -193,9 +199,6 @@ namespace libtorrent
 				}
 			}
 
-			if (resources == 0 || sum_max == 0)
-				return;
-
 			if (sum_max <= resources)
 			{
 				// it turns out that there's no competition for resources

include/libtorrent/peer_connection.hpp

@@ -562,14 +562,7 @@ namespace libtorrent
 		// these are true when there's a asynchronous write
 		// or read operation running.
 		bool m_writing;
-		// this is the number of bytes sent to the socket last
-		// time it was invoked. This is compared against the
-		// bytes_transferred in the callback function that tells
-		// how much actually was sent. Then the quota can be
-		// corrected according to the actual number of bytes sent
-		int m_last_write_size;
 		bool m_reading;
-		int m_last_read_size;		
 
 		// if set to true, this peer will always prefer
 		// to request entire pieces, rather than blocks.

include/libtorrent/resource_request.hpp

@@ -54,6 +54,7 @@ namespace libtorrent
 			, min(0)
 			, max(0)
 			, given(0)
+			, leftovers(0)
 		{}
 
 		resource_request(int used_, int min_, int max_, int given_)
@@ -61,16 +62,18 @@ namespace libtorrent
 			, min(min_)
 			, max(max_)
 			, given(given_)
+			, leftovers(0)
 		{}
 
 		int left() const
 		{
 			assert(given <= max);
 			assert(given >= min);
-			if (used < 0 && (given - used < 0))
-				return boost::integer_traits<int>::const_max;
-			return given - used;
+			assert(used >= 0);
+			return (std::max)(given - used, 0);
 		}
+		
+		void reset() { used = leftovers; }
 
 		static const int inf = boost::integer_traits<int>::const_max;
 
@@ -84,6 +87,11 @@ namespace libtorrent
 
 		// Reply: Okay, you're allowed to use this amount (a compromise):
 		int given;
+
+		// this is the amount of resources that exceeded the
+		// given limit. When the used field is reset (after resources
+		// have been distributed), it is reset to this number.
+		int leftovers;
 	};
 }
 

include/libtorrent/session.hpp

@@ -60,6 +60,7 @@ POSSIBILITY OF SUCH DAMAGE.
 #include "libtorrent/version.hpp"
 #include "libtorrent/fingerprint.hpp"
 
+#include "libtorrent/resource_request.hpp"
 
 #if !defined(NDEBUG) && defined(_MSC_VER)
 #	include <float.h>
@@ -214,6 +215,12 @@ namespace libtorrent
 		std::auto_ptr<alert> pop_alert();
 		void set_severity_level(alert::severity_t s);
 
+      // Resource management used for global limits.
+      resource_request m_ul_bandwidth_quota;
+		resource_request m_dl_bandwidth_quota;
+		resource_request m_uploads_quota;
+		resource_request m_connections_quota;
+
 	private:
 
 		// data shared between the main thread

include/libtorrent/torrent.hpp

@@ -382,8 +382,6 @@ namespace libtorrent
 
 		// LOGGING
 #if defined(TORRENT_VERBOSE_LOGGING) || defined(TORRENT_LOGGING)
-		logger* spawn_logger(const char* title);
-
 		virtual void debug_log(const std::string& line);
 #endif
 
@@ -397,7 +395,7 @@ namespace libtorrent
 
 		// this will distribute the given upload/download
 		// quotas and number of connections, among the peers
-		void distribute_resources();
+		void distribute_resources(float tick_interval);
 
 		resource_request m_ul_bandwidth_quota;
 		resource_request m_dl_bandwidth_quota;
@@ -613,6 +611,14 @@ namespace libtorrent
 		// total_done - m_initial_done <= total_payload_download
 		size_type m_initial_done;
 #endif
+
+#ifdef TORRENT_LOGGING
+		boost::shared_ptr<logger> m_log;
+		int m_second_count;
+		
+		int m_ul_history[10];
+		int m_dl_history[10];
+#endif
 	};
 
 	inline boost::posix_time::ptime torrent::next_announce() const

src/allocate_resources.cpp

@@ -163,6 +163,11 @@ namespace libtorrent
 		{
 			return *p.second;
 		}
+
+		session& deref(session* p)
+		{
+			return *p;
+		}
 	}
 
 	void allocate_resources(
@@ -196,6 +201,23 @@ namespace libtorrent
 			, new_iter(c.end(), &aux::pick_peer2)
 			, res);
 	}
+
+   void allocate_resources(
+		int resources
+		, std::vector<session*>& _sessions
+		, resource_request session::* res)
+   {
+      typedef std::vector<session*>::iterator orig_iter;
+      typedef session* in_param;
+		typedef boost::transform_iterator<session& (*)(in_param), orig_iter> new_iter;
+
+      aux::allocate_resources_impl(
+			resources
+			, new_iter(_sessions.begin(), &aux::deref)
+			, new_iter(_sessions.end(), &aux::deref)
+			, res);
+   }
+
 #endif
 
 } // namespace libtorrent

src/peer_connection.cpp

@@ -113,9 +113,7 @@ namespace libtorrent
 		, m_connecting(true)
 		, m_queued(true)
 		, m_writing(false)
-		, m_last_write_size(0)
 		, m_reading(false)
-		, m_last_read_size(0)
 		, m_prefer_whole_pieces(false)
 		, m_request_large_blocks(false)
 		, m_refs(0)
@@ -204,9 +202,7 @@ namespace libtorrent
 		, m_connecting(false)
 		, m_queued(false)
 		, m_writing(false)
-		, m_last_write_size(0)
 		, m_reading(false)
-		, m_last_read_size(0)
 		, m_prefer_whole_pieces(false)
 		, m_request_large_blocks(false)
 		, m_refs(0)
@@ -459,8 +455,8 @@ namespace libtorrent
 
 	void peer_connection::reset_upload_quota()
 	{
-		m_ul_bandwidth_quota.used = 0;
-		m_dl_bandwidth_quota.used = 0;
+		m_ul_bandwidth_quota.reset();
+		m_dl_bandwidth_quota.reset();
 		assert(m_ul_bandwidth_quota.left() >= 0);
 		assert(m_dl_bandwidth_quota.left() >= 0);
 		setup_send();
@@ -1581,9 +1577,6 @@ namespace libtorrent
 		}
 
 		m_statistics.second_tick(tick_interval);
-		m_ul_bandwidth_quota.used = std::min(
-			(int)ceil(statistics().upload_rate())
-			, m_ul_bandwidth_quota.given);
 
 		// If the client sends more data
 		// we send it data faster, otherwise, slower.
@@ -1599,13 +1592,17 @@ namespace libtorrent
 			// have an unlimited upload rate
 			if(send_buffer_size() > 0
 				|| (!m_requests.empty() && !is_choked()))
+			{
 				m_ul_bandwidth_quota.max = resource_request::inf;
+			}
 			else
+			{
 				m_ul_bandwidth_quota.max = m_ul_bandwidth_quota.min;
+			}
 		}
 		else
 		{
-			size_type bias = 0x10000+2*t->block_size() + m_free_upload;
+			size_type bias = 0x10000 + 2 * t->block_size() + m_free_upload;
 
 			double break_even_time = 15; // seconds.
 			size_type have_uploaded = m_statistics.total_payload_upload();
@@ -1619,7 +1616,7 @@ namespace libtorrent
 				soon_downloaded = (size_type)(soon_downloaded*(double)t->ratio());
 
 			double upload_speed_limit = (soon_downloaded - have_uploaded
-				                         + bias) / break_even_time;
+				+ bias) / break_even_time;
 
 			upload_speed_limit = std::min(upload_speed_limit,
 				(double)std::numeric_limits<int>::max());
@@ -1630,9 +1627,6 @@ namespace libtorrent
 		if (m_ul_bandwidth_quota.given > m_ul_bandwidth_quota.max)
 			m_ul_bandwidth_quota.given = m_ul_bandwidth_quota.max;
 
-		if (m_ul_bandwidth_quota.used > m_ul_bandwidth_quota.given)
-			m_ul_bandwidth_quota.used = m_ul_bandwidth_quota.given;
-
 		fill_send_buffer();
 /*
 		size_type diff = share_diff();
@@ -1757,8 +1751,6 @@ namespace libtorrent
 				, bind(&peer_connection::on_send_data, self(), _1, _2));
 
 			m_writing = true;
-			m_last_write_size = amount_to_send;
-			m_ul_bandwidth_quota.used += m_last_write_size;
 		}
 	}
 
@@ -1785,8 +1777,6 @@ namespace libtorrent
 		m_socket->async_read_some(asio::buffer(&m_recv_buffer[m_recv_pos]
 			, max_receive), bind(&peer_connection::on_receive_data, self(), _1, _2));
 		m_reading = true;
-		m_last_read_size = max_receive;
-		m_dl_bandwidth_quota.used += max_receive;
 		assert(m_dl_bandwidth_quota.used <= m_dl_bandwidth_quota.given);
 	}
 
@@ -1841,12 +1831,9 @@ namespace libtorrent
 		INVARIANT_CHECK;
 
 		assert(m_reading);
-		assert(m_last_read_size > 0);
-		assert(m_last_read_size >= int(bytes_transferred));
 		m_reading = false;
 		// correct the dl quota usage, if not all of the buffer was actually read
-		m_dl_bandwidth_quota.used -= m_last_read_size - bytes_transferred;
-		m_last_read_size = 0;
+		m_dl_bandwidth_quota.used += bytes_transferred;
 
 		if (error)
 		{
@@ -1986,8 +1973,8 @@ namespace libtorrent
 		(*m_ses.m_logger) << "COMPLETED: " << m_remote.address().to_string() << "\n";
 #endif
 
-		m_connecting = false;
 		m_ses.connection_completed(self());
+		m_connecting = false;
 		on_connected();
 		setup_send();
 	}
@@ -2017,11 +2004,9 @@ namespace libtorrent
 		INVARIANT_CHECK;
 
 		assert(m_writing);
-		assert(m_last_write_size > 0);
 		m_writing = false;
 		// correct the ul quota usage, if not all of the buffer was sent
-		m_ul_bandwidth_quota.used -= m_last_write_size - bytes_transferred;
-		m_last_write_size = 0;
+		m_ul_bandwidth_quota.used += bytes_transferred;
 		m_write_pos += bytes_transferred;
 
 		if (error)

src/session_impl.cpp

@@ -996,7 +996,7 @@ namespace libtorrent { namespace detail
 #ifndef NDEBUG
 			i->second->check_invariant();
 #endif
-			i->second->distribute_resources();
+			i->second->distribute_resources(tick_interval);
 		}
 	}
 	catch (std::exception& exc)

src/torrent.cpp

@@ -187,6 +187,27 @@ namespace
 
 		peer_id const& pid;
 	};
+	
+#ifdef TORRENT_LOGGING
+	void print_legend(boost::shared_ptr<logger> l)
+	{
+		(*l) << "time, seconds\n"
+			<< "bytes sent\n"
+			<< "bytes sent 10 seconds mean\n"
+			<< "hard send quota, bytes\n"
+			<< "soft send quota, bytes\n"
+			<< "excess bytes sent\n"
+			<< "bytes received\n"
+			<< "bytes received 10 seconds mean\n"
+			<< "hard receive quota, bytes\n"
+			<< "soft receive quota, bytes\n"
+			<< "excess bytes received\n"
+			<< "num peers\n"
+			
+			<< "\n";
+	}
+#endif
+
 }
 
 namespace libtorrent
@@ -243,6 +264,15 @@ namespace libtorrent
 	{
 #ifndef NDEBUG
 		m_initial_done = 0;
+#endif
+#ifdef TORRENT_LOGGING
+		m_log = ses.create_log("torrent_"
+			+ boost::lexical_cast<std::string>(tf.info_hash())
+			, m_ses.listen_port(), true);
+		print_legend(m_log);
+		m_second_count = 0;
+		std::fill_n(m_ul_history, 10, 0);
+		std::fill_n(m_dl_history, 10, 0);
 #endif
 		INVARIANT_CHECK;
 
@@ -344,6 +374,17 @@ namespace libtorrent
 #ifndef NDEBUG
 		m_initial_done = 0;
 #endif
+
+#ifdef TORRENT_LOGGING
+		m_log = ses.create_log("torrent_"
+			+ boost::lexical_cast<std::string>(info_hash)
+			, m_ses.listen_port(), true);
+		print_legend(m_log);
+		m_second_count = 0;
+		std::fill_n(m_ul_history, 10, 0);
+		std::fill_n(m_dl_history, 10, 0);
+#endif
+
 		INVARIANT_CHECK;
 
 		if (name) m_name.reset(new std::string(name));
@@ -1802,7 +1843,7 @@ namespace libtorrent
 		m_stat.second_tick(tick_interval);
 	}
 
-	void torrent::distribute_resources()
+	void torrent::distribute_resources(float tick_interval)
 	{
 		INVARIANT_CHECK;
 
@@ -1824,6 +1865,9 @@ namespace libtorrent
 			ul_used += i->second->m_ul_bandwidth_quota.used;
 			dl_used += i->second->m_dl_bandwidth_quota.used;
 		}
+		
+		assert(ul_used == m_ul_bandwidth_quota.used);
+		assert(dl_used == m_dl_bandwidth_quota.used);
 
 		m_excess_ul += ul_used - m_ul_bandwidth_quota.given;
 		m_excess_dl += dl_used - m_dl_bandwidth_quota.given;
@@ -1831,21 +1875,42 @@ namespace libtorrent
 		m_excess_ul = std::max(m_excess_ul, 0);
 		m_excess_dl = std::max(m_excess_dl, 0);
 
-#if defined(TORRENT_VERBOSE_LOGGING) || defined(TORRENT_LOGGING)
-/*		for (peer_iterator i = m_connections.begin();
-			i != m_connections.end(); ++i)
-		{
-			(*m_ses.m_logger) << i->second->remote() << " (" << i->second->m_dl_bandwidth_quota.given << ", "
-				<< i->second->m_dl_bandwidth_quota.used << ") ";
-		}
-*/
-		(*m_ses.m_logger) << m_excess_ul << " " << m_excess_dl << "\n";
-#endif
+		int ul_to_distribute = std::max(int((m_ul_bandwidth_quota.given
+			- m_excess_ul) * 1.6f), 0);
+		int dl_to_distribute = std::max(int((m_dl_bandwidth_quota.given
+			- m_excess_dl) * 1.6f), 0);
 
-		int ul_to_distribute = int((m_ul_bandwidth_quota.given
-			- m_excess_ul) * 1.3f);
-		int dl_to_distribute = int((m_dl_bandwidth_quota.given
-			- m_excess_dl) * 1.3f);
+#ifdef TORRENT_LOGGING
+		std::copy(m_ul_history + 1, m_ul_history + 10, m_ul_history);
+		m_ul_history[9] = ul_used;
+		std::copy(m_dl_history + 1, m_dl_history + 10, m_dl_history);
+		m_dl_history[9] = dl_used;
+
+		size_type mean_ul = 0;
+		size_type mean_dl = 0;
+		for (int i = 0; i < 10; ++i)
+		{
+			mean_ul += m_ul_history[i];
+			mean_dl += m_dl_history[i];
+		}
+		mean_ul /= 10;
+		mean_dl /= 10;
+
+		(*m_log) << m_second_count++ << "\t"
+			<< ul_used << "\t"
+			<< mean_ul << "\t"
+			<< m_ul_bandwidth_quota.given << "\t"
+			<< ul_to_distribute << "\t"
+			<< m_excess_ul << "\t"
+			<< dl_used << "\t"
+			<< mean_dl << "\t"
+			<< m_dl_bandwidth_quota.given << "\t"
+			<< dl_to_distribute << "\t"
+			<< m_excess_dl << "\t"
+			<< num_peers() << "\t"
+			<< "\n";
+
+#endif
 
 		// distribute allowed upload among the peers
 		allocate_resources(ul_to_distribute
@@ -1866,8 +1931,6 @@ namespace libtorrent
 			= m_connections.begin(); i != m_connections.end(); ++i)
 		{
 			i->second->reset_upload_quota();
-			assert(i->second->m_dl_bandwidth_quota.used
-				<= i->second->m_dl_bandwidth_quota.given);
 		}
 	}