the file pool has been moved to the session and its size can be controlled via session_settings. Added untested option to allow multiple connections from the same IP.

ChangeLog

@@ -1,3 +1,5 @@
+	* moved the file_pool into session. The number of open files is now
+	  limited per session.
 	* fixed uninitialized private flag in torrent_info
 	* fixed long standing issue with file.cpp on windows. Replaced the low level
 	  io functions used on windows.

docs/examples.html

@@ -180,6 +180,7 @@ int main(int argc, char* argv[])
 #include "libtorrent/file.hpp"
 #include "libtorrent/storage.hpp"
 #include "libtorrent/hasher.hpp"
+#include "libtorrent/file_pool.hpp"
 
 #include <boost/filesystem/operations.hpp>
 #include <boost/filesystem/path.hpp>
@@ -188,7 +189,10 @@ int main(int argc, char* argv[])
 using namespace boost::filesystem;
 using namespace libtorrent;
 
-void add_files(torrent_info& t, path const& p, path const& l)
+void add_files(
+        torrent_info& t
+        , path const& p
+        , path const& l)
 {
         path f(p / l);
         if (is_directory(f))
@@ -199,10 +203,7 @@ void add_files(torrent_info& t, path const& p, path const& l)
         else
         {
                 std::cerr << "adding \"" << l.string() << "\"\n";
-                file fi(f, file::in);
-                fi.seek(0, file::end);
-                libtorrent::size_type size = fi.tell();
-                t.add_file(l, size);
+                t.add_file(l, file_size(f));
         }
 }
 
@@ -211,6 +212,8 @@ int main(int argc, char* argv[])
         using namespace libtorrent;
         using namespace boost::filesystem;
 
+        path::default_name_check(no_check);
+
         if (argc != 4)
         {
                 std::cerr << "usage: make_torrent <output torrent-file> "
@@ -218,13 +221,11 @@ int main(int argc, char* argv[])
                 return 1;
         }
 
-        boost::filesystem::path::default_name_check(native);
-
         try
         {
                 torrent_info t;
-                path full_path = initial_path() / path(argv[3]);
-                ofstream out(initial_path() / path(argv[1]), std::ios_base::binary);
+                path full_path = complete(path(argv[3]));
+                ofstream out(complete(path(argv[1])), std::ios_base::binary);
 
                 int piece_size = 256 * 1024;
                 char const* creator_str = "libtorrent";
@@ -232,7 +233,8 @@ int main(int argc, char* argv[])
                 add_files(t, full_path.branch_path(), full_path.leaf());
                 t.set_piece_size(piece_size);
 
-                storage st(t, full_path.branch_path());
+                file_pool fp;
+                storage st(t, full_path.branch_path(), fp);
                 t.add_tracker(argv[2]);
 
                 // calculate the hash for all pieces
@@ -251,8 +253,8 @@ int main(int argc, char* argv[])
                 // create the torrent and print it to out
                 entry e = t.create_torrent();
                 libtorrent::bencode(std::ostream_iterator<char>(out), e);
-        }
-        catch (std::exception& e)
+                }
+                catch (std::exception& e)
         {
                 std::cerr << e.what() << "\n";
         }

docs/examples.rst

@@ -158,22 +158,26 @@ Shows how to create a torrent from a directory tree::
 	#include <fstream>
 	#include <iterator>
 	#include <iomanip>
-
+	
 	#include "libtorrent/entry.hpp"
 	#include "libtorrent/bencode.hpp"
 	#include "libtorrent/torrent_info.hpp"
 	#include "libtorrent/file.hpp"
 	#include "libtorrent/storage.hpp"
 	#include "libtorrent/hasher.hpp"
-
+	#include "libtorrent/file_pool.hpp"
+	
 	#include <boost/filesystem/operations.hpp>
 	#include <boost/filesystem/path.hpp>
 	#include <boost/filesystem/fstream.hpp>
-
+	
 	using namespace boost::filesystem;
 	using namespace libtorrent;
-
-	void add_files(torrent_info& t, path const& p, path const& l)
+	
+	void add_files(
+		torrent_info& t
+		, path const& p
+		, path const& l)
 	{
 		path f(p / l);
 		if (is_directory(f))
@@ -184,42 +188,40 @@ Shows how to create a torrent from a directory tree::
 		else
 		{
 			std::cerr << "adding \"" << l.string() << "\"\n";
-			file fi(f, file::in);
-			fi.seek(0, file::end);
-			libtorrent::size_type size = fi.tell();
-			t.add_file(l, size);
+			t.add_file(l, file_size(f));
 		}
 	}
-
+	
 	int main(int argc, char* argv[])
 	{
 		using namespace libtorrent;
 		using namespace boost::filesystem;
-
+	
+		path::default_name_check(no_check);
+	
 		if (argc != 4)
 		{
 			std::cerr << "usage: make_torrent <output torrent-file> "
 				"<announce url> <file or directory to create torrent from>\n";
 			return 1;
 		}
-
-		boost::filesystem::path::default_name_check(native);
-
+	
 		try
 		{
 			torrent_info t;
-			path full_path = initial_path() / path(argv[3]);
-			ofstream out(initial_path() / path(argv[1]), std::ios_base::binary);
-
+			path full_path = complete(path(argv[3]));
+			ofstream out(complete(path(argv[1])), std::ios_base::binary);
+	
 			int piece_size = 256 * 1024;
 			char const* creator_str = "libtorrent";
-
+	
 			add_files(t, full_path.branch_path(), full_path.leaf());
 			t.set_piece_size(piece_size);
-
-			storage st(t, full_path.branch_path());
+	
+			file_pool fp;
+			storage st(t, full_path.branch_path(), fp);
 			t.add_tracker(argv[2]);
-
+	
 			// calculate the hash for all pieces
 			int num = t.num_pieces();
 			std::vector<char> buf(piece_size);
@@ -230,19 +232,19 @@ Shows how to create a torrent from a directory tree::
 				t.set_hash(i, h.final());
 				std::cerr << (i+1) << "/" << num << "\r";
 			}
-
+	
 			t.set_creator(creator_str);
-
+	
 			// create the torrent and print it to out
 			entry e = t.create_torrent();
 			libtorrent::bencode(std::ostream_iterator<char>(out), e);
-		}
-		catch (std::exception& e)
+			}
+			catch (std::exception& e)
 		{
 			std::cerr << e.what() << "\n";
 		}
-
+	
 		return 0;
 	}
-
+	
 

docs/manual.html

@@ -1913,6 +1913,8 @@ struct session_settings
         int peer_timeout;
         int urlseed_timeout;
         int urlseed_pipeline_size;
+        int file_pool_size;
+        bool allow_multiple_connections_per_ip;
 };
 </pre>
 <p><tt class="docutils literal"><span class="pre">proxy_ip</span></tt> may be a hostname or ip to a http proxy to use. If this is
@@ -1973,6 +1975,20 @@ url seeds. This value defaults to 20 seconds.</p>
 using persistent connections to HTTP 1.1 servers, the client is allowed to
 send more requests before the first response is received. This number controls
 the number of outstanding requests to use with url-seeds. Default is 5.</p>
+<p><tt class="docutils literal"><span class="pre">file_pool_size</span></tt> is the the upper limit on the total number of files this
+session will keep open. The reason why files are left open at all is that
+some anti virus software hooks on every file close, and scans the file for
+viruses. deferring the closing of the files will be the difference between
+a usable system and a completely hogged down system. Most operating systems
+also has a limit on the total number of file descriptors a process may have
+open. It is usually a good idea to find this limit and set the number of
+connections and the number of files limits so their sum is slightly below it.</p>
+<p><tt class="docutils literal"><span class="pre">allow_multiple_connections_per_ip</span></tt> determines if connections from the
+same IP address as existing connections should be rejected or not. Multiple
+connections from the same IP address is not allowed by default, to prevent
+abusive behavior by peers. It may be useful to allow such connections in
+cases where simulations are run on the same machie, and all peers in a
+swarm has the same IP address.</p>
 </div>
 <div class="section" id="ip-filter">
 <h1><a name="ip-filter">ip_filter</a></h1>

docs/manual.rst

@@ -1894,6 +1894,8 @@ that will be sent to the tracker. The user-agent is a good way to identify your
 		int peer_timeout;
 		int urlseed_timeout;
 		int urlseed_pipeline_size;
+		int file_pool_size;
+		bool allow_multiple_connections_per_ip;
 	};
 
 ``proxy_ip`` may be a hostname or ip to a http proxy to use. If this is
@@ -1970,6 +1972,22 @@ using persistent connections to HTTP 1.1 servers, the client is allowed to
 send more requests before the first response is received. This number controls
 the number of outstanding requests to use with url-seeds. Default is 5.
 
+``file_pool_size`` is the the upper limit on the total number of files this
+session will keep open. The reason why files are left open at all is that
+some anti virus software hooks on every file close, and scans the file for
+viruses. deferring the closing of the files will be the difference between
+a usable system and a completely hogged down system. Most operating systems
+also has a limit on the total number of file descriptors a process may have
+open. It is usually a good idea to find this limit and set the number of
+connections and the number of files limits so their sum is slightly below it.
+
+``allow_multiple_connections_per_ip`` determines if connections from the
+same IP address as existing connections should be rejected or not. Multiple
+connections from the same IP address is not allowed by default, to prevent
+abusive behavior by peers. It may be useful to allow such connections in
+cases where simulations are run on the same machie, and all peers in a
+swarm has the same IP address.
+
 ip_filter
 =========
 

examples/make_torrent.cpp

@@ -1,6 +1,6 @@
 /*
 
-Copyright (c) 2003, Arvid Norberg
+Copyright (c) 2006, Arvid Norberg
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
@@ -41,6 +41,7 @@ POSSIBILITY OF SUCH DAMAGE.
 #include "libtorrent/file.hpp"
 #include "libtorrent/storage.hpp"
 #include "libtorrent/hasher.hpp"
+#include "libtorrent/file_pool.hpp"
 
 #include <boost/filesystem/operations.hpp>
 #include <boost/filesystem/path.hpp>
@@ -94,7 +95,8 @@ int main(int argc, char* argv[])
 		add_files(t, full_path.branch_path(), full_path.leaf());
 		t.set_piece_size(piece_size);
 
-		storage st(t, full_path.branch_path());
+		file_pool fp;
+		storage st(t, full_path.branch_path(), fp);
 		t.add_tracker(argv[2]);
 
 		// calculate the hash for all pieces

include/libtorrent/aux_/session_impl.hpp

@@ -75,6 +75,7 @@ POSSIBILITY OF SUCH DAMAGE.
 #include "libtorrent/session_status.hpp"
 #include "libtorrent/session.hpp"
 #include "libtorrent/stat.hpp"
+#include "libtorrent/file_pool.hpp"
 
 namespace libtorrent
 {
@@ -338,6 +339,11 @@ namespace libtorrent
 			// this is used to know if the client is behind
 			// NAT or not.
 			bool m_incoming_connection;
+			
+			// the file pool that all storages in this session's
+			// torrents uses. It sets a limit on the number of
+			// open files by this session.
+			file_pool m_files;
 
 			// does the actual disconnections
 			// that are queued up in m_disconnect_peer

include/libtorrent/file_pool.hpp

@@ -63,10 +63,11 @@ namespace libtorrent
 
 	struct file_pool
 	{
-		file_pool(int size): m_size(size) {}
+		file_pool(int size = 40): m_size(size) {}
 
 		boost::shared_ptr<file> open_file(void* st, fs::path const& p, file::open_mode m);
 		void release(void* st);
+		void resize(int size);
 
 	private:
 		int m_size;

include/libtorrent/session_settings.hpp

@@ -56,6 +56,8 @@ namespace libtorrent
 			, peer_timeout(120)
 			, urlseed_timeout(20)
 			, urlseed_pipeline_size(5)
+			, file_pool_size(40)
+			, allow_multiple_connections_per_ip(false)
 		{}
 
 		std::string proxy_ip;
@@ -130,6 +132,24 @@ namespace libtorrent
 		
 		// controls the pipelining size of url-seeds
 		int urlseed_pipeline_size;
+		
+		// sets the upper limit on the total number of files this
+		// session will keep open. The reason why files are
+		// left open at all is that some anti virus software
+		// hooks on every file close, and scans the file for
+		// viruses. deferring the closing of the files will
+		// be the difference between a usable system and
+		// a completely hogged down system. Most operating
+		// systems also has a limit on the total number of
+		// file descriptors a process may have open. It is
+		// usually a good idea to find this limit and set the
+		// number of connections and the number of files
+		// limits so their sum is slightly below it.
+		int file_pool_size;
+		
+		// false to not allow multiple connections from the same
+		// IP address. true will allow it.
+		bool allow_multiple_connections_per_ip;
 	};
 	
 #ifndef TORRENT_DISABLE_DHT

include/libtorrent/storage.hpp

@@ -61,6 +61,7 @@ namespace libtorrent
 	}
 
 	class session;
+	class file_pool;
 
 #if defined(_WIN32) && defined(UNICODE)
 
@@ -91,7 +92,8 @@ namespace libtorrent
 	public:
 		storage(
 			const torrent_info& info
-			, const boost::filesystem::path& path);
+			, const boost::filesystem::path& path
+			, file_pool& fp);
 
 		void swap(storage&);
 
@@ -125,7 +127,8 @@ namespace libtorrent
 
 		piece_manager(
 			const torrent_info& info
-			, const boost::filesystem::path& path);
+			, const boost::filesystem::path& path
+			, file_pool& fp);
 
 		~piece_manager();
 

src/bt_peer_connection.cpp

@@ -1203,7 +1203,7 @@ namespace libtorrent
 #ifdef TORRENT_VERBOSE_LOGGING
 					(*m_logger) << " received invalid info_hash\n";
 #endif
-					throw std::runtime_error("invalid info-hash in handshake");
+					throw protocol_error("invalid info-hash in handshake");
 				}
 			}
 
@@ -1279,13 +1279,6 @@ namespace libtorrent
 			if (m_supports_extensions) write_extensions();
 #endif
 
-/*
-			if (!m_active)
-			{
-				m_attached_to_torrent = true;
-				assert(m_torrent->get_policy().has_connection(this));
-			}
-*/
 			m_state = read_packet_size;
 			reset_recv_buffer(4);
 		}

src/file_pool.cpp

@@ -101,4 +101,23 @@ namespace libtorrent
 		kt.erase(start, end);
 	}
 
+	void file_pool::resize(int size)
+	{
+		assert(size > 0);
+		if (size == m_size) return;
+		m_size = size;
+		if (int(m_files.size()) <= m_size) return;
+
+		// close the least recently used files
+		typedef nth_index<file_set, 1>::type lru_view;
+		lru_view& lt = get<1>(m_files);
+		lru_view::iterator i = lt.begin();
+		while (int(m_files.size()) > m_size)
+		{
+			// the first entry in this view is the least recently used
+			assert(lt.size() == 1 || (i->last_use <= boost::next(i)->last_use));
+			lt.erase(i++);
+		}
+	}
+
 }

src/policy.cpp

@@ -871,21 +871,9 @@ namespace libtorrent
 	void policy::new_connection(peer_connection& c)
 	{
 		assert(!c.is_local());
-/*
-#ifndef NDEBUG
-		// avoid the invariant check to fail
-		peer p(tcp::endpoint("0.0.0.0", 0), peer::not_connectable);
-		p.connection = &c;
-		m_peers.push_back(p);
-#endif
-*/
+
 		INVARIANT_CHECK;
-/*
-#ifndef NDEBUG
-		// avoid the invariant check to fail
-		m_peers.erase(m_peers.end() - 1);
-#endif
-*/
+
 		// if the connection comes from the tracker,
 		// it's probably just a NAT-check. Ignore the
 		// num connections constraint then.
@@ -908,11 +896,19 @@ namespace libtorrent
 		}
 #endif
 
-		std::vector<peer>::iterator i = std::find_if(
-			m_peers.begin()
-			, m_peers.end()
-			, match_peer_ip(c.remote()));
+		std::vector<peer>::iterator i;
 
+		if (m_torrent->settings().allow_multiple_connections_per_ip)
+		{
+			i = m_peers.end();
+		}
+		else
+		{
+			i = std::find_if(
+				m_peers.begin()
+				, m_peers.end()
+				, match_peer_ip(c.remote()));
+		}
 
 		if (i != m_peers.end())
 		{
@@ -1322,7 +1318,7 @@ namespace libtorrent
 		return std::find_if(
 			m_peers.begin()
 			, m_peers.end()
-			, match_peer_ip(c->remote())) != m_peers.end();
+			, match_peer_connection(*c)) != m_peers.end();
 	}
 
 	void policy::check_invariant() const

src/session_impl.cpp

@@ -475,6 +475,7 @@ namespace libtorrent { namespace detail
 		, m_max_connections(-1)
 		, m_half_open_limit(-1)
 		, m_incoming_connection(false)
+		, m_files(40)
 		, m_last_tick(microsec_clock::universal_time())
 		, m_timer(m_selector)
 		, m_checker_impl(*this)
@@ -575,6 +576,7 @@ namespace libtorrent { namespace detail
 	{
 		mutex_t::scoped_lock l(m_mutex);
 		m_settings = s;
+		m_files.resize(m_settings.file_pool_size);
 		// replace all occurances of '\n' with ' '.
 		std::string::iterator i = m_settings.user_agent.begin();
 		while ((i = std::find(i, m_settings.user_agent.end(), '\n'))

src/storage.cpp

@@ -342,9 +342,10 @@ namespace libtorrent
 	class storage::impl : public thread_safe_storage, boost::noncopyable
 	{
 	public:
-		impl(torrent_info const& info, path const& path)
+		impl(torrent_info const& info, path const& path, file_pool& fp)
 			: thread_safe_storage(info.num_pieces())
 			, info(info)
+			, files(fp)
 		{
 			save_path = complete(path);
 			assert(save_path.is_complete());
@@ -354,6 +355,7 @@ namespace libtorrent
 			: thread_safe_storage(x.info.num_pieces())
 			, info(x.info)
 			, save_path(x.save_path)
+			, files(x.files)
 		{}
 
 		~impl()
@@ -363,13 +365,15 @@ namespace libtorrent
 
 		torrent_info const& info;
 		path save_path;
-		static file_pool files;
+		// the file pool is typically stored in
+		// the session, to make all storage
+		// instances use the same pool
+		file_pool& files;
 	};
 
-	file_pool storage::impl::files(40);
-
-	storage::storage(torrent_info const& info, path const& path)
-		: m_pimpl(new impl(info, path))
+	storage::storage(torrent_info const& info, path const& path
+		, file_pool& fp)
+		: m_pimpl(new impl(info, path, fp))
 	{
 		assert(info.begin_files() != info.end_files());
 	}
@@ -735,7 +739,8 @@ namespace libtorrent
 
 		impl(
 			torrent_info const& info
-			, path const& path);
+			, path const& path
+			, file_pool& fp);
 
 		bool check_fastresume(
 			aux::piece_checker_data& d
@@ -884,8 +889,9 @@ namespace libtorrent
 
 	piece_manager::impl::impl(
 		torrent_info const& info
-		, path const& save_path)
-		: m_storage(info, save_path)
+		, path const& save_path
+		, file_pool& fp)
+		: m_storage(info, save_path, fp)
 		, m_compact_mode(false)
 		, m_fill_mode(true)
 		, m_info(info)
@@ -897,8 +903,9 @@ namespace libtorrent
 
 	piece_manager::piece_manager(
 		torrent_info const& info
-		, path const& save_path)
-		: m_pimpl(new impl(info, save_path))
+		, path const& save_path
+		, file_pool& fp)
+		: m_pimpl(new impl(info, save_path, fp))
 	{
 	}
 

src/torrent.cpp

@@ -431,7 +431,7 @@ namespace libtorrent
 		assert(m_torrent_file.total_size() >= 0);
 
 		m_have_pieces.resize(m_torrent_file.num_pieces(), false);
-		m_storage.reset(new piece_manager(m_torrent_file, m_save_path));
+		m_storage.reset(new piece_manager(m_torrent_file, m_save_path, m_ses.m_files));
 		m_block_size = calculate_block_size(m_torrent_file, m_default_block_size);
 		m_picker.reset(new piece_picker(
 			static_cast<int>(m_torrent_file.piece_length() / m_block_size)