From 51e3261dd012522f2b0dce9576a8d062ce70e042 Mon Sep 17 00:00:00 2001 From: Arvid Norberg Date: Tue, 14 Nov 2006 15:53:38 +0000 Subject: [PATCH] the file pool has been moved to the session and its size can be controlled via session_settings. Added untested option to allow multiple connections from the same IP. --- ChangeLog | 2 + docs/examples.html | 26 ++++++----- docs/examples.rst | 56 ++++++++++++------------ docs/manual.html | 16 +++++++ docs/manual.rst | 18 ++++++++ examples/make_torrent.cpp | 6 ++- include/libtorrent/aux_/session_impl.hpp | 6 +++ include/libtorrent/file_pool.hpp | 3 +- include/libtorrent/session_settings.hpp | 20 +++++++++ include/libtorrent/storage.hpp | 7 ++- src/bt_peer_connection.cpp | 9 +--- src/file_pool.cpp | 19 ++++++++ src/policy.cpp | 34 +++++++------- src/session_impl.cpp | 2 + src/storage.cpp | 29 +++++++----- src/torrent.cpp | 2 +- 16 files changed, 172 insertions(+), 83 deletions(-) diff --git a/ChangeLog b/ChangeLog index ee25379a2..a25c3e08e 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,5 @@ + * moved the file_pool into session. The number of open files is now + limited per session. * fixed uninitialized private flag in torrent_info * fixed long standing issue with file.cpp on windows. Replaced the low level io functions used on windows. diff --git a/docs/examples.html b/docs/examples.html index 57aec57c3..f1a962d71 100644 --- a/docs/examples.html +++ b/docs/examples.html @@ -180,6 +180,7 @@ int main(int argc, char* argv[]) #include "libtorrent/file.hpp" #include "libtorrent/storage.hpp" #include "libtorrent/hasher.hpp" +#include "libtorrent/file_pool.hpp" #include <boost/filesystem/operations.hpp> #include <boost/filesystem/path.hpp> @@ -188,7 +189,10 @@ int main(int argc, char* argv[]) using namespace boost::filesystem; using namespace libtorrent; -void add_files(torrent_info& t, path const& p, path const& l) +void add_files( + torrent_info& t + , path const& p + , path const& l) { path f(p / l); if (is_directory(f)) @@ -199,10 +203,7 @@ void add_files(torrent_info& t, path const& p, path const& l) else { std::cerr << "adding \"" << l.string() << "\"\n"; - file fi(f, file::in); - fi.seek(0, file::end); - libtorrent::size_type size = fi.tell(); - t.add_file(l, size); + t.add_file(l, file_size(f)); } } @@ -211,6 +212,8 @@ int main(int argc, char* argv[]) using namespace libtorrent; using namespace boost::filesystem; + path::default_name_check(no_check); + if (argc != 4) { std::cerr << "usage: make_torrent <output torrent-file> " @@ -218,13 +221,11 @@ int main(int argc, char* argv[]) return 1; } - boost::filesystem::path::default_name_check(native); - try { torrent_info t; - path full_path = initial_path() / path(argv[3]); - ofstream out(initial_path() / path(argv[1]), std::ios_base::binary); + path full_path = complete(path(argv[3])); + ofstream out(complete(path(argv[1])), std::ios_base::binary); int piece_size = 256 * 1024; char const* creator_str = "libtorrent"; @@ -232,7 +233,8 @@ int main(int argc, char* argv[]) add_files(t, full_path.branch_path(), full_path.leaf()); t.set_piece_size(piece_size); - storage st(t, full_path.branch_path()); + file_pool fp; + storage st(t, full_path.branch_path(), fp); t.add_tracker(argv[2]); // calculate the hash for all pieces @@ -251,8 +253,8 @@ int main(int argc, char* argv[]) // create the torrent and print it to out entry e = t.create_torrent(); libtorrent::bencode(std::ostream_iterator<char>(out), e); - } - catch (std::exception& e) + } + catch (std::exception& e) { std::cerr << e.what() << "\n"; } diff --git a/docs/examples.rst b/docs/examples.rst index 2845a29f9..4f459a1ba 100644 --- a/docs/examples.rst +++ b/docs/examples.rst @@ -158,22 +158,26 @@ Shows how to create a torrent from a directory tree:: #include #include #include - + #include "libtorrent/entry.hpp" #include "libtorrent/bencode.hpp" #include "libtorrent/torrent_info.hpp" #include "libtorrent/file.hpp" #include "libtorrent/storage.hpp" #include "libtorrent/hasher.hpp" - + #include "libtorrent/file_pool.hpp" + #include #include #include - + using namespace boost::filesystem; using namespace libtorrent; - - void add_files(torrent_info& t, path const& p, path const& l) + + void add_files( + torrent_info& t + , path const& p + , path const& l) { path f(p / l); if (is_directory(f)) @@ -184,42 +188,40 @@ Shows how to create a torrent from a directory tree:: else { std::cerr << "adding \"" << l.string() << "\"\n"; - file fi(f, file::in); - fi.seek(0, file::end); - libtorrent::size_type size = fi.tell(); - t.add_file(l, size); + t.add_file(l, file_size(f)); } } - + int main(int argc, char* argv[]) { using namespace libtorrent; using namespace boost::filesystem; - + + path::default_name_check(no_check); + if (argc != 4) { std::cerr << "usage: make_torrent " " \n"; return 1; } - - boost::filesystem::path::default_name_check(native); - + try { torrent_info t; - path full_path = initial_path() / path(argv[3]); - ofstream out(initial_path() / path(argv[1]), std::ios_base::binary); - + path full_path = complete(path(argv[3])); + ofstream out(complete(path(argv[1])), std::ios_base::binary); + int piece_size = 256 * 1024; char const* creator_str = "libtorrent"; - + add_files(t, full_path.branch_path(), full_path.leaf()); t.set_piece_size(piece_size); - - storage st(t, full_path.branch_path()); + + file_pool fp; + storage st(t, full_path.branch_path(), fp); t.add_tracker(argv[2]); - + // calculate the hash for all pieces int num = t.num_pieces(); std::vector buf(piece_size); @@ -230,19 +232,19 @@ Shows how to create a torrent from a directory tree:: t.set_hash(i, h.final()); std::cerr << (i+1) << "/" << num << "\r"; } - + t.set_creator(creator_str); - + // create the torrent and print it to out entry e = t.create_torrent(); libtorrent::bencode(std::ostream_iterator(out), e); - } - catch (std::exception& e) + } + catch (std::exception& e) { std::cerr << e.what() << "\n"; } - + return 0; } - + diff --git a/docs/manual.html b/docs/manual.html index 131134acc..b04e2465c 100755 --- a/docs/manual.html +++ b/docs/manual.html @@ -1913,6 +1913,8 @@ struct session_settings int peer_timeout; int urlseed_timeout; int urlseed_pipeline_size; + int file_pool_size; + bool allow_multiple_connections_per_ip; };

proxy_ip may be a hostname or ip to a http proxy to use. If this is @@ -1973,6 +1975,20 @@ url seeds. This value defaults to 20 seconds.

using persistent connections to HTTP 1.1 servers, the client is allowed to send more requests before the first response is received. This number controls the number of outstanding requests to use with url-seeds. Default is 5.

+

file_pool_size is the the upper limit on the total number of files this +session will keep open. The reason why files are left open at all is that +some anti virus software hooks on every file close, and scans the file for +viruses. deferring the closing of the files will be the difference between +a usable system and a completely hogged down system. Most operating systems +also has a limit on the total number of file descriptors a process may have +open. It is usually a good idea to find this limit and set the number of +connections and the number of files limits so their sum is slightly below it.

+

allow_multiple_connections_per_ip determines if connections from the +same IP address as existing connections should be rejected or not. Multiple +connections from the same IP address is not allowed by default, to prevent +abusive behavior by peers. It may be useful to allow such connections in +cases where simulations are run on the same machie, and all peers in a +swarm has the same IP address.

ip_filter

diff --git a/docs/manual.rst b/docs/manual.rst index 832566afa..509ca24f9 100755 --- a/docs/manual.rst +++ b/docs/manual.rst @@ -1894,6 +1894,8 @@ that will be sent to the tracker. The user-agent is a good way to identify your int peer_timeout; int urlseed_timeout; int urlseed_pipeline_size; + int file_pool_size; + bool allow_multiple_connections_per_ip; }; ``proxy_ip`` may be a hostname or ip to a http proxy to use. If this is @@ -1970,6 +1972,22 @@ using persistent connections to HTTP 1.1 servers, the client is allowed to send more requests before the first response is received. This number controls the number of outstanding requests to use with url-seeds. Default is 5. +``file_pool_size`` is the the upper limit on the total number of files this +session will keep open. The reason why files are left open at all is that +some anti virus software hooks on every file close, and scans the file for +viruses. deferring the closing of the files will be the difference between +a usable system and a completely hogged down system. Most operating systems +also has a limit on the total number of file descriptors a process may have +open. It is usually a good idea to find this limit and set the number of +connections and the number of files limits so their sum is slightly below it. + +``allow_multiple_connections_per_ip`` determines if connections from the +same IP address as existing connections should be rejected or not. Multiple +connections from the same IP address is not allowed by default, to prevent +abusive behavior by peers. It may be useful to allow such connections in +cases where simulations are run on the same machie, and all peers in a +swarm has the same IP address. + ip_filter ========= diff --git a/examples/make_torrent.cpp b/examples/make_torrent.cpp index 0c9c0e33a..d136120cd 100755 --- a/examples/make_torrent.cpp +++ b/examples/make_torrent.cpp @@ -1,6 +1,6 @@ /* -Copyright (c) 2003, Arvid Norberg +Copyright (c) 2006, Arvid Norberg All rights reserved. Redistribution and use in source and binary forms, with or without @@ -41,6 +41,7 @@ POSSIBILITY OF SUCH DAMAGE. #include "libtorrent/file.hpp" #include "libtorrent/storage.hpp" #include "libtorrent/hasher.hpp" +#include "libtorrent/file_pool.hpp" #include #include @@ -94,7 +95,8 @@ int main(int argc, char* argv[]) add_files(t, full_path.branch_path(), full_path.leaf()); t.set_piece_size(piece_size); - storage st(t, full_path.branch_path()); + file_pool fp; + storage st(t, full_path.branch_path(), fp); t.add_tracker(argv[2]); // calculate the hash for all pieces diff --git a/include/libtorrent/aux_/session_impl.hpp b/include/libtorrent/aux_/session_impl.hpp index c24e23be7..d3d87fa5a 100644 --- a/include/libtorrent/aux_/session_impl.hpp +++ b/include/libtorrent/aux_/session_impl.hpp @@ -75,6 +75,7 @@ POSSIBILITY OF SUCH DAMAGE. #include "libtorrent/session_status.hpp" #include "libtorrent/session.hpp" #include "libtorrent/stat.hpp" +#include "libtorrent/file_pool.hpp" namespace libtorrent { @@ -338,6 +339,11 @@ namespace libtorrent // this is used to know if the client is behind // NAT or not. bool m_incoming_connection; + + // the file pool that all storages in this session's + // torrents uses. It sets a limit on the number of + // open files by this session. + file_pool m_files; // does the actual disconnections // that are queued up in m_disconnect_peer diff --git a/include/libtorrent/file_pool.hpp b/include/libtorrent/file_pool.hpp index 054f9ccfe..3bdd9765b 100644 --- a/include/libtorrent/file_pool.hpp +++ b/include/libtorrent/file_pool.hpp @@ -63,10 +63,11 @@ namespace libtorrent struct file_pool { - file_pool(int size): m_size(size) {} + file_pool(int size = 40): m_size(size) {} boost::shared_ptr open_file(void* st, fs::path const& p, file::open_mode m); void release(void* st); + void resize(int size); private: int m_size; diff --git a/include/libtorrent/session_settings.hpp b/include/libtorrent/session_settings.hpp index 80d328611..82b8a96e7 100644 --- a/include/libtorrent/session_settings.hpp +++ b/include/libtorrent/session_settings.hpp @@ -56,6 +56,8 @@ namespace libtorrent , peer_timeout(120) , urlseed_timeout(20) , urlseed_pipeline_size(5) + , file_pool_size(40) + , allow_multiple_connections_per_ip(false) {} std::string proxy_ip; @@ -130,6 +132,24 @@ namespace libtorrent // controls the pipelining size of url-seeds int urlseed_pipeline_size; + + // sets the upper limit on the total number of files this + // session will keep open. The reason why files are + // left open at all is that some anti virus software + // hooks on every file close, and scans the file for + // viruses. deferring the closing of the files will + // be the difference between a usable system and + // a completely hogged down system. Most operating + // systems also has a limit on the total number of + // file descriptors a process may have open. It is + // usually a good idea to find this limit and set the + // number of connections and the number of files + // limits so their sum is slightly below it. + int file_pool_size; + + // false to not allow multiple connections from the same + // IP address. true will allow it. + bool allow_multiple_connections_per_ip; }; #ifndef TORRENT_DISABLE_DHT diff --git a/include/libtorrent/storage.hpp b/include/libtorrent/storage.hpp index be2980816..f29b54c73 100755 --- a/include/libtorrent/storage.hpp +++ b/include/libtorrent/storage.hpp @@ -61,6 +61,7 @@ namespace libtorrent } class session; + class file_pool; #if defined(_WIN32) && defined(UNICODE) @@ -91,7 +92,8 @@ namespace libtorrent public: storage( const torrent_info& info - , const boost::filesystem::path& path); + , const boost::filesystem::path& path + , file_pool& fp); void swap(storage&); @@ -125,7 +127,8 @@ namespace libtorrent piece_manager( const torrent_info& info - , const boost::filesystem::path& path); + , const boost::filesystem::path& path + , file_pool& fp); ~piece_manager(); diff --git a/src/bt_peer_connection.cpp b/src/bt_peer_connection.cpp index 6f97b519c..add52b311 100755 --- a/src/bt_peer_connection.cpp +++ b/src/bt_peer_connection.cpp @@ -1203,7 +1203,7 @@ namespace libtorrent #ifdef TORRENT_VERBOSE_LOGGING (*m_logger) << " received invalid info_hash\n"; #endif - throw std::runtime_error("invalid info-hash in handshake"); + throw protocol_error("invalid info-hash in handshake"); } } @@ -1279,13 +1279,6 @@ namespace libtorrent if (m_supports_extensions) write_extensions(); #endif -/* - if (!m_active) - { - m_attached_to_torrent = true; - assert(m_torrent->get_policy().has_connection(this)); - } -*/ m_state = read_packet_size; reset_recv_buffer(4); } diff --git a/src/file_pool.cpp b/src/file_pool.cpp index 130ca7501..9bdb10397 100644 --- a/src/file_pool.cpp +++ b/src/file_pool.cpp @@ -101,4 +101,23 @@ namespace libtorrent kt.erase(start, end); } + void file_pool::resize(int size) + { + assert(size > 0); + if (size == m_size) return; + m_size = size; + if (int(m_files.size()) <= m_size) return; + + // close the least recently used files + typedef nth_index::type lru_view; + lru_view& lt = get<1>(m_files); + lru_view::iterator i = lt.begin(); + while (int(m_files.size()) > m_size) + { + // the first entry in this view is the least recently used + assert(lt.size() == 1 || (i->last_use <= boost::next(i)->last_use)); + lt.erase(i++); + } + } + } diff --git a/src/policy.cpp b/src/policy.cpp index e165cc1c6..4a3910eca 100755 --- a/src/policy.cpp +++ b/src/policy.cpp @@ -871,21 +871,9 @@ namespace libtorrent void policy::new_connection(peer_connection& c) { assert(!c.is_local()); -/* -#ifndef NDEBUG - // avoid the invariant check to fail - peer p(tcp::endpoint("0.0.0.0", 0), peer::not_connectable); - p.connection = &c; - m_peers.push_back(p); -#endif -*/ + INVARIANT_CHECK; -/* -#ifndef NDEBUG - // avoid the invariant check to fail - m_peers.erase(m_peers.end() - 1); -#endif -*/ + // if the connection comes from the tracker, // it's probably just a NAT-check. Ignore the // num connections constraint then. @@ -908,11 +896,19 @@ namespace libtorrent } #endif - std::vector::iterator i = std::find_if( - m_peers.begin() - , m_peers.end() - , match_peer_ip(c.remote())); + std::vector::iterator i; + if (m_torrent->settings().allow_multiple_connections_per_ip) + { + i = m_peers.end(); + } + else + { + i = std::find_if( + m_peers.begin() + , m_peers.end() + , match_peer_ip(c.remote())); + } if (i != m_peers.end()) { @@ -1322,7 +1318,7 @@ namespace libtorrent return std::find_if( m_peers.begin() , m_peers.end() - , match_peer_ip(c->remote())) != m_peers.end(); + , match_peer_connection(*c)) != m_peers.end(); } void policy::check_invariant() const diff --git a/src/session_impl.cpp b/src/session_impl.cpp index 33afc8d07..cfbe5c945 100755 --- a/src/session_impl.cpp +++ b/src/session_impl.cpp @@ -475,6 +475,7 @@ namespace libtorrent { namespace detail , m_max_connections(-1) , m_half_open_limit(-1) , m_incoming_connection(false) + , m_files(40) , m_last_tick(microsec_clock::universal_time()) , m_timer(m_selector) , m_checker_impl(*this) @@ -575,6 +576,7 @@ namespace libtorrent { namespace detail { mutex_t::scoped_lock l(m_mutex); m_settings = s; + m_files.resize(m_settings.file_pool_size); // replace all occurances of '\n' with ' '. std::string::iterator i = m_settings.user_agent.begin(); while ((i = std::find(i, m_settings.user_agent.end(), '\n')) diff --git a/src/storage.cpp b/src/storage.cpp index ca01fb0ff..508be5534 100755 --- a/src/storage.cpp +++ b/src/storage.cpp @@ -342,9 +342,10 @@ namespace libtorrent class storage::impl : public thread_safe_storage, boost::noncopyable { public: - impl(torrent_info const& info, path const& path) + impl(torrent_info const& info, path const& path, file_pool& fp) : thread_safe_storage(info.num_pieces()) , info(info) + , files(fp) { save_path = complete(path); assert(save_path.is_complete()); @@ -354,6 +355,7 @@ namespace libtorrent : thread_safe_storage(x.info.num_pieces()) , info(x.info) , save_path(x.save_path) + , files(x.files) {} ~impl() @@ -363,13 +365,15 @@ namespace libtorrent torrent_info const& info; path save_path; - static file_pool files; + // the file pool is typically stored in + // the session, to make all storage + // instances use the same pool + file_pool& files; }; - file_pool storage::impl::files(40); - - storage::storage(torrent_info const& info, path const& path) - : m_pimpl(new impl(info, path)) + storage::storage(torrent_info const& info, path const& path + , file_pool& fp) + : m_pimpl(new impl(info, path, fp)) { assert(info.begin_files() != info.end_files()); } @@ -735,7 +739,8 @@ namespace libtorrent impl( torrent_info const& info - , path const& path); + , path const& path + , file_pool& fp); bool check_fastresume( aux::piece_checker_data& d @@ -884,8 +889,9 @@ namespace libtorrent piece_manager::impl::impl( torrent_info const& info - , path const& save_path) - : m_storage(info, save_path) + , path const& save_path + , file_pool& fp) + : m_storage(info, save_path, fp) , m_compact_mode(false) , m_fill_mode(true) , m_info(info) @@ -897,8 +903,9 @@ namespace libtorrent piece_manager::piece_manager( torrent_info const& info - , path const& save_path) - : m_pimpl(new impl(info, save_path)) + , path const& save_path + , file_pool& fp) + : m_pimpl(new impl(info, save_path, fp)) { } diff --git a/src/torrent.cpp b/src/torrent.cpp index 70a0c0606..8e214914d 100755 --- a/src/torrent.cpp +++ b/src/torrent.cpp @@ -431,7 +431,7 @@ namespace libtorrent assert(m_torrent_file.total_size() >= 0); m_have_pieces.resize(m_torrent_file.num_pieces(), false); - m_storage.reset(new piece_manager(m_torrent_file, m_save_path)); + m_storage.reset(new piece_manager(m_torrent_file, m_save_path, m_ses.m_files)); m_block_size = calculate_block_size(m_torrent_file, m_default_block_size); m_picker.reset(new piece_picker( static_cast(m_torrent_file.piece_length() / m_block_size)