make bottled http requests response size limit configurable and bump default to 2 MiB

ChangeLog

@@ -5,6 +5,7 @@
 	* fix uTP edge case where udp socket buffer fills up
 	* fix nagle implementation in uTP
 
+	* raised the limit for bottled http downloads to 2 MiB
 	* add support for magnet links and URLs in python example client
 	* fixed typo in python bindings' add_torrent_params
 	* introduce a way to add built-in plugins from python

bindings/python/src/session_settings.cpp

@@ -172,6 +172,8 @@ void bind_session_settings()
         .def_readwrite("enable_incoming_utp", &session_settings::enable_incoming_utp)
         .def_readwrite("ssl_listen", &session_settings::ssl_listen)
         .def_readwrite("tracker_backoff", &session_settings::tracker_backoff)
+        .def_readwrite("ban_web_seeds", &session_settings::ban_web_seeds)
+        .def_readwrite("max_http_recv_buffer_size", &session_settings::max_http_recv_buffer_size)
     ;
 
     enum_<proxy_settings::proxy_type>("proxy_type")

docs/manual.rst

@@ -4601,6 +4601,7 @@ session_settings
 		int tracker_backoff;
 
 		bool ban_web_seeds;
+		int max_http_recv_buffer_size;
 	};
 
 ``version`` is automatically set to the libtorrent version you're using
@@ -5495,6 +5496,11 @@ trackers.
 ``ban_web_seeds`` enables banning web seeds. By default, web seeds that send
 corrupt data are banned.
 
+``max_http_recv_buffer_size`` specifies the max number of bytes to receive into
+RAM buffers when downloading stuff over HTTP. Specifically when specifying a
+URL to a .torrent file when adding a torrent or when announcing to an HTTP
+tracker. The default is 2 MiB.
+
 pe_settings
 ===========
 

include/libtorrent/http_connection.hpp

@@ -63,6 +63,8 @@ namespace libtorrent
 
 struct http_connection;
 class connection_queue;
+
+const int default_max_bottled_buffer_size = 2*1024*1024;
 	
 typedef boost::function<void(error_code const&
 	, http_parser const&, char const* data, int size, http_connection&)> http_handler;
@@ -73,10 +75,13 @@ typedef boost::function<void(http_connection&, std::list<tcp::endpoint>&)> http_
 
 // when bottled, the last two arguments to the handler
 // will always be 0
-struct TORRENT_EXTRA_EXPORT http_connection : boost::enable_shared_from_this<http_connection>, boost::noncopyable
+struct TORRENT_EXTRA_EXPORT http_connection
+	: boost::enable_shared_from_this<http_connection>
+	, boost::noncopyable
 {
 	http_connection(io_service& ios, connection_queue& cc
 		, http_handler const& handler, bool bottled = true
+		, int max_bottled_buffer_size = default_max_bottled_buffer_size
 		, http_connect_handler const& ch = http_connect_handler()
 		, http_filter_handler const& fh = http_filter_handler()
 #ifdef TORRENT_USE_OPENSSL
@@ -152,11 +157,16 @@ private:
 	time_duration m_completion_timeout;
 	ptime m_last_receive;
 	ptime m_start_time;
+	
 	// bottled means that the handler is called once, when
 	// everything is received (and buffered in memory).
 	// non bottled means that once the headers have been
 	// received, data is streamed to the handler
 	bool m_bottled;
+
+	// maximum size of bottled buffer
+	int m_max_bottled_buffer_size;
+	
 	// set to true the first time the handler is called
 	bool m_called;
 	std::string m_hostname;

include/libtorrent/session_settings.hpp

@@ -933,6 +933,10 @@ namespace libtorrent
 
 		// when true, web seeds sending bad data will be banned
 		bool ban_web_seeds;
+		
+		// http_connection maximum receive buffer size
+		// limits torrent file size for URL torrents
+		int max_http_recv_buffer_size;
 	};
 
 #ifndef TORRENT_DISABLE_DHT

src/http_connection.cpp

@@ -49,10 +49,10 @@ POSSIBILITY OF SUCH DAMAGE.
 
 namespace libtorrent {
 
-enum { max_bottled_buffer = 1024 * 1024 };
-
 http_connection::http_connection(io_service& ios, connection_queue& cc
-	, http_handler const& handler, bool bottled
+	, http_handler const& handler
+	, bool bottled
+	, int max_bottled_buffer_size
 	, http_connect_handler const& ch
 	, http_filter_handler const& fh
 #ifdef TORRENT_USE_OPENSSL
@@ -72,6 +72,7 @@ http_connection::http_connection(io_service& ios, connection_queue& cc
 	, m_last_receive(time_now())
 	, m_start_time(time_now())
 	, m_bottled(bottled)
+	, m_max_bottled_buffer_size(max_bottled_buffer_size)
 	, m_called(false)
 #ifdef TORRENT_USE_OPENSSL
 	, m_ssl_ctx(ssl_ctx)
@@ -625,7 +626,7 @@ void http_connection::callback(error_code e, char const* data, int size)
 		if ((encoding == "gzip" || encoding == "x-gzip") && size > 0 && data)
 		{
 			std::string error;
-			if (inflate_gzip(data, size, buf, max_bottled_buffer, error))
+			if (inflate_gzip(data, size, buf, m_max_bottled_buffer_size, error))
 			{
 				if (m_handler) m_handler(errors::http_failed_decompress, m_parser, data, size, *this);
 				close();
@@ -837,8 +838,8 @@ void http_connection::on_read(error_code const& e
 	}
 
 	if (int(m_recvbuffer.size()) == m_read_pos)
-		m_recvbuffer.resize((std::min)(m_read_pos + 2048, int(max_bottled_buffer)));
-	if (m_read_pos == max_bottled_buffer)
+		m_recvbuffer.resize((std::min)(m_read_pos + 2048, m_max_bottled_buffer_size));
+	if (m_read_pos == m_max_bottled_buffer_size)
 	{
 		callback(asio::error::eof);
 		close();

src/http_tracker_connection.cpp

@@ -214,7 +214,7 @@ namespace libtorrent
 
 		m_tracker_connection.reset(new http_connection(m_ios, m_cc
 			, boost::bind(&http_tracker_connection::on_response, self(), _1, _2, _3, _4)
-			, true
+			, true, settings.max_http_recv_buffer_size
 			, boost::bind(&http_tracker_connection::on_connect, self(), _1)
 			, boost::bind(&http_tracker_connection::on_filter, self(), _1, _2)
 #ifdef TORRENT_USE_OPENSSL

src/session.cpp

@@ -174,6 +174,9 @@ namespace libtorrent
 
 		// disallow the buffer size to grow for the uTP socket
 		set.utp_dynamic_sock_buf = false;
+		
+		// max 'bottled' http receive buffer/url torrent size
+		set.max_http_recv_buffer_size = 1024 * 1024;
 
 		return set;
 	}
@@ -292,6 +295,9 @@ namespace libtorrent
 		// allow the buffer size to grow for the uTP socket
 		set.utp_dynamic_sock_buf = true;
 
+		// max 'bottled' http receive buffer/url torrent size
+		set.max_http_recv_buffer_size = 6 * 1024 * 1024;
+
 		return set;
 	}
 
@@ -1283,6 +1289,7 @@ namespace libtorrent
 		, ssl_listen(4433)
 		, tracker_backoff(250)
 		, ban_web_seeds(true)
+		, max_http_recv_buffer_size(2*1024*1024)
 	{}
 
 	session_settings::~session_settings() {}

src/session_impl.cpp

@@ -445,6 +445,8 @@ namespace aux {
 		TORRENT_SETTING(boolean, lock_files)
 		TORRENT_SETTING(integer, ssl_listen)
 		TORRENT_SETTING(integer, tracker_backoff)
+		TORRENT_SETTING(boolean, ban_web_seeds)
+		TORRENT_SETTING(integer, max_http_recv_buffer_size)
 	};
 
 #undef TORRENT_SETTING

src/torrent.cpp

@@ -908,9 +908,12 @@ namespace libtorrent
 		TORRENT_ASSERT(!m_url.empty());
 		TORRENT_ASSERT(!m_torrent_file->is_valid());
 		boost::shared_ptr<http_connection> conn(
-			new http_connection(m_ses.m_io_service, m_ses.m_half_open
+			new http_connection(m_ses.m_io_service, m_ses.m_half_open			        
 				, boost::bind(&torrent::on_torrent_download, shared_from_this()
-					, _1, _2, _3, _4)));
+					, _1, _2, _3, _4)
+				, true //bottled
+				, m_ses.settings().max_http_recv_buffer_size //bottled buffer size
+				));
 		conn->get(m_url, seconds(30), 0, 0, 5, m_ses.m_settings.user_agent);
 		set_state(torrent_status::downloading_metadata);
 	}

src/upnp.cpp

@@ -699,7 +699,7 @@ void upnp::update_map(rootdevice& d, int i, mutex::scoped_lock& l)
 		if (d.upnp_connection) d.upnp_connection->close();
 		d.upnp_connection.reset(new http_connection(m_io_service
 			, m_cc, boost::bind(&upnp::on_upnp_map_response, self(), _1, _2
-			, boost::ref(d), i, _5), true
+			, boost::ref(d), i, _5), true, default_max_bottled_buffer_size
 			, boost::bind(&upnp::create_port_mapping, self(), _1, boost::ref(d), i)));
 
 		d.upnp_connection->start(d.hostname, to_string(d.port).elems
@@ -710,7 +710,7 @@ void upnp::update_map(rootdevice& d, int i, mutex::scoped_lock& l)
 		if (d.upnp_connection) d.upnp_connection->close();
 		d.upnp_connection.reset(new http_connection(m_io_service
 			, m_cc, boost::bind(&upnp::on_upnp_unmap_response, self(), _1, _2
-			, boost::ref(d), i, _5), true
+			, boost::ref(d), i, _5), true, default_max_bottled_buffer_size
 			, boost::bind(&upnp::delete_port_mapping, self(), boost::ref(d), i)));
 		d.upnp_connection->start(d.hostname, to_string(d.port).elems
 			, seconds(10), 1);
@@ -958,7 +958,7 @@ void upnp::on_upnp_xml(error_code const& e
 
 	d.upnp_connection.reset(new http_connection(m_io_service
 		, m_cc, boost::bind(&upnp::on_upnp_get_ip_address_response, self(), _1, _2
-		, boost::ref(d), _5), true
+		, boost::ref(d), _5), true, default_max_bottled_buffer_size 
 		, boost::bind(&upnp::get_ip_address, self(), boost::ref(d))));
 	d.upnp_connection->start(d.hostname, to_string(d.port).elems
 		, seconds(10), 1);

test/test_http_connection.cpp

@@ -116,7 +116,7 @@ void run_test(std::string const& url, int size, int status, int connected
 		<< " error: " << (ec?ec->message():"no error") << std::endl;
 
 	boost::shared_ptr<http_connection> h(new http_connection(ios, cq
-		, &::http_handler, true, &::http_connect_handler));
+		, &::http_handler, true, 1024*1024, &::http_connect_handler));
 	h->get(url, seconds(1), 0, &ps);
 	ios.reset();
 	error_code e;